<!doctype html>



  


<html class="theme-next muse use-motion" lang="zh-Hans">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>









<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />




  
  
  
  

  
    
    
  

  

  

  

  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.1" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="OpenStack," />








  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=5.1.1" />






<meta name="description" content="概述&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;Neutron 的设计目标是实现“网络即服务”，为了达到这一目标，在设计上遵循了基于“软件定义网络”实现网络虚拟化的原则，在实现上充分利用了 Linux 系统上的各种网络相关的技术。 &amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;理解了 Linux">
<meta name="keywords" content="OpenStack">
<meta property="og:type" content="article">
<meta property="og:title" content="深入理解 Neutron -- OpenStack 网络实现">
<meta property="og:url" content="https://hcldirgit.github.io/2017/09/03/OpenStack/4. 深入理解 Neutron -- OpenStack 网络实现/index.html">
<meta property="og:site_name" content="失落的乐章">
<meta property="og:description" content="概述&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;Neutron 的设计目标是实现“网络即服务”，为了达到这一目标，在设计上遵循了基于“软件定义网络”实现网络虚拟化的原则，在实现上充分利用了 Linux 系统上的各种网络相关的技术。 &amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;理解了 Linux">
<meta property="og:locale" content="zh-Hans">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/01.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/02.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/03.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/04.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/05.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/06.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/07.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/08.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/09.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/09.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/11.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/12.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/13.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/14.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/15.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/16.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/17.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/18.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/19.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/20.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/21.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/22.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/23.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/24.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/25.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/26.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/27.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/28.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/29.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/30.png?raw=true">
<meta property="og:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/31.png?raw=true">
<meta property="og:updated_time" content="2017-09-02T17:27:29.937Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="深入理解 Neutron -- OpenStack 网络实现">
<meta name="twitter:description" content="概述&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;Neutron 的设计目标是实现“网络即服务”，为了达到这一目标，在设计上遵循了基于“软件定义网络”实现网络虚拟化的原则，在实现上充分利用了 Linux 系统上的各种网络相关的技术。 &amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;&amp;#160;理解了 Linux">
<meta name="twitter:image" content="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/01.png?raw=true">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Muse',
    sidebar: {"position":"left","display":"post","offset":12,"offset_float":0,"b2t":false,"scrollpercent":false},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="https://hcldirgit.github.io/2017/09/03/OpenStack/4. 深入理解 Neutron -- OpenStack 网络实现/"/>





  <title>深入理解 Neutron -- OpenStack 网络实现 | 失落的乐章</title>
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  




<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
            (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
          m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
  ga('create', '85*****1', 'auto');
  ga('send', 'pageview');
</script>


  <script type="text/javascript">
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?87980c**************99ec5e26fb5";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>











  
  
    
  

  <div class="container sidebar-position-left page-post-detail ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">失落的乐章</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle">技术面前，永远都是学生。</p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-message">
          <a href="/message" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-external-link"></i> <br />
            
            留言
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
    <link itemprop="mainEntityOfPage" href="https://hcldirgit.github.io/2017/09/03/OpenStack/4. 深入理解 Neutron -- OpenStack 网络实现/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="失落的乐章">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/0.png">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="失落的乐章">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">深入理解 Neutron -- OpenStack 网络实现</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2017-09-03T02:09:22+08:00">
                2017-09-03
              </time>
            

            

            
          </span>

          

          
            
          

          
          

          

          

          

        </div>
      </header>
    

    <div class="post-body" itemprop="articleBody">

      
      

      
        <h2 id="概述"><a href="#概述" class="headerlink" title="概述"></a>概述</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;Neutron 的设计目标是实现“网络即服务”，为了达到这一目标，在设计上遵循了基于“软件定义网络”实现网络虚拟化的原则，在实现上充分利用了 Linux 系统上的各种网络相关的技术。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;理解了 Linux 系统上的这些概念将有利于快速理解 Neutron 的原理和实现。</p>
<h3 id="涉及的-Linux-网络技术"><a href="#涉及的-Linux-网络技术" class="headerlink" title="涉及的 Linux 网络技术"></a>涉及的 Linux 网络技术</h3><ul>
<li>bridge：网桥，Linux中用于表示一个能连接不同网络设备的虚拟设备，linux中传统实现的网桥类似一个hub设备，而ovs管理的网桥一般类似交换机。</li>
<li>br-int：bridge-integration，综合网桥，常用于表示实现主要内部网络功能的网桥。</li>
<li>br-ex：bridge-external，外部网桥，通常表示负责跟外部网络通信的网桥。</li>
<li>GRE：General Routing Encapsulation，一种通过封装来实现隧道的方式。在openstack中一般是基于L3的gre，即original pkt/GRE/IP/Ethernet</li>
<li>VETH：虚拟ethernet接口，通常以pair的方式出现，一端发出的网包，会被另一端接收，可以形成两个网桥之间的通道。</li>
<li>qvb：neutron veth, Linux Bridge-side</li>
<li>qvo：neutron veth, OVS-side</li>
<li>TAP设备：模拟一个二层的网络设备，可以接受和发送二层网包。</li>
<li>TUN设备：模拟一个三层的网络设备，可以接受和发送三层网包。</li>
<li>iptables：Linux 上常见的实现安全策略的防火墙软件。</li>
<li>Vlan：虚拟 Lan，同一个物理 Lan 下用标签实现隔离，可用标号为1-4094。</li>
<li>VXLAN：一套利用 UDP 协议作为底层传输协议的 Overlay 实现。一般认为作为 VLan 技术的延伸或替代者。</li>
<li>namespace：用来实现隔离的一套机制，不同 namespace 中的资源之间彼此不可见。</li>
</ul>
<h2 id="基本概念"><a href="#基本概念" class="headerlink" title="基本概念"></a>基本概念</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;Neutron管理下面的实体：</p>
<ul>
<li>网络：隔离的 L2 域，可以是虚拟、逻辑或交换。</li>
<li>子网：隔离的 L3 域，IP 地址块。其中每个机器有一个 IP，同一个子网的主机彼此 L3 可见。</li>
<li>端口：网络上虚拟、逻辑或交换端口。 所有这些实体都是虚拟的，拥有自动生成的唯一标示id，支持CRUD功能，并在数据库中跟踪记录状态。</li>
</ul>
<h3 id="网络"><a href="#网络" class="headerlink" title="网络"></a>网络</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;隔离的 L2 广播域，一般是创建它的用户所有。用户可以拥有多个网络。网络是最基础的，子网和端口都需要关联到网络上。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;网络上可以有多个子网。同一个网络上的主机一般可以通过交换机或路由器连通起来。</p>
<h3 id="子网"><a href="#子网" class="headerlink" title="子网"></a>子网</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;隔离的 L3 域，子网代表了一组分配了 IP 的虚拟机。每个子网必须有一个 CIDR 和关联到一个网络。IP 可以从 CIDR 或者用户指定池中选取。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;子网可能会有一个网关、一组 DNS 和主机路由。不同子网之间 L3 是互相不可见的，必须通过一个三层网关（即路由器）经过 L3 上进行通信。</p>
<h3 id="端口"><a href="#端口" class="headerlink" title="端口"></a>端口</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以进出流量的接口，往往绑定上若干 MAC 地址和 IP 地址，以进行寻址。一般为虚拟交换机上的虚拟接口。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;虚拟机挂载网卡到端口上，通过端口访问网络。当端口有 IP 的时候，意味着它属于某个子网。</p>
<h3 id="抽象系统架构"><a href="#抽象系统架构" class="headerlink" title="抽象系统架构"></a>抽象系统架构</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;无论哪种具体的网络虚拟化实现，一个简化和抽象后的系统架构可以表述为下图所示。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/01.png?raw=true" alt="01"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在启用 DVR 特性（J 版本以后支持）之前，所有流量（东西向、南北向）都需要经过网络节点的转发；DVR 特性则允许东西向流量和带有 Floating IP 的南北向流量不经过网络节点的转发，直接从计算节点的外部网络出去。</p>
<h2 id="GRE-模式"><a href="#GRE-模式" class="headerlink" title="GRE 模式"></a>GRE 模式</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;下图给出了在OpenStack中网络实现的一个简化的架构示意。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;一般的，OpenStack中网络实现包括vlan、gre、vxlan 等模式，此处以gre模式为例。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/02.png?raw=true" alt="02"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在OpenStack中，所有网络有关的逻辑管理均在Network节点中实现，例如DNS、DHCP以及路由等。Compute节点上只需要对所部属的虚拟机提供基本的网络功能支持，包括隔离不同租户的虚拟机和进行一些基本的安全策略管理（即security group）。</p>
<h3 id="计算节点"><a href="#计算节点" class="headerlink" title="计算节点"></a>计算节点</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;以抽象系统架构的图表为例，Compute 节点上包括两台虚拟机 VM1 和 VM2，分别经过一个网桥（如 qbr-XXX）连接到 br-int 网桥上。br-int 网桥再经过 br-tun 网桥（物理网络是 GRE 实现）连接到物理主机外部网络。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对于物理网络通过 vlan 来隔离的情况，则一般会存在一个 br-eth 网桥，替代 br-tun 网桥。</p>
<h3 id="qbr"><a href="#qbr" class="headerlink" title="qbr"></a>qbr</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 VM1 中，虚拟机的网卡实际上连接到了物理机的一个 TAP 设备（即 A，常见名称如 tap-XXX）上，A 则进一步通过VETH pair（A-B）连接到网桥 qbr-XXX 的端口 vnet0（端口 B）上，之后再通过 VETH pair（C-D）连到br-int网桥上。一般C的名字格式为 qvb-XXX，而 D 的名字格式为 qvo-XXX。注意它们的名称除了前缀外，后面的 id 都是一样的，表示位于同一个虚拟机网络到物理机网络的连接上。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;之所以 TAP 设备 A 没有直接连接到网桥br-int上，是因为 OpenStack 需要通过 iptables 实现 security group 的安全策略功能。目前 openvswitch 并不支持应用 iptables 规则的 Tap 设备。<br>因为 qbr 的存在主要是为了辅助 iptables 来实现 security group功能，有时候也被称为安全网桥。详见 security group 部分的分析。</p>
<h3 id="br-int"><a href="#br-int" class="headerlink" title="br-int"></a>br-int</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;一个典型的 br-int 的端口如下所示：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div></pre></td><td class="code"><pre><div class="line">ovs-vsctl show</div><div class="line">Bridge br-int</div><div class="line">    Port <span class="string">"qvo-XXX"</span></div><div class="line">        tag: 1</div><div class="line">        Interface <span class="string">"qvo-XXX"</span></div><div class="line">    Port patch-tun</div><div class="line">        Interface patch-tun</div><div class="line">            <span class="built_in">type</span>: patch</div><div class="line">            options: &#123;peer=patch-int&#125;</div><div class="line">    Port br-int</div><div class="line">        Interface br-int</div><div class="line">            <span class="built_in">type</span>: internal</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，</p>
<ul>
<li>br-int 为内部端口。</li>
<li>patch-tun（即端口E，端口号为1）连接到 br-tun 上，实现到外部网络的隧道。</li>
<li>qvo-XXX（即端口D，端口号为2）带有 tag1，说明这个口是一个1号 vlan 的 access 端口。虚拟机发出的从该端口到达br-int的网包将被自动带上vlan tag 1，而其他带有 vlan tag 1 的网包则可以在去掉 vlan tag 后从该端口发出（即 vlan access 端口）。这个 vlan tag 是用来实现不同网络相互隔离的，比如租户创建一个网络（neutron net-create），则会被分配一个唯一的 vlan tag。</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-int 在 GRE 模式中作为一个 NORMAL 交换机使用，因此有效规则只有一条正常转发。如果两个在同一主机上的 vm 属于同一个 tenant 的（同一个 vlan tag），则它们之间的通信只需要经过 br-int 即可。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-int</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=10727.864s, table=0, n_packets=198, n_bytes=17288, idle_age=13, priority=1 actions=NORMAL</div></pre></td></tr></table></figure>
<h3 id="br-tun"><a href="#br-tun" class="headerlink" title="br-tun"></a>br-tun</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;一个典型的 br-tun 上的端口类似：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div></pre></td><td class="code"><pre><div class="line">Bridge br-tun</div><div class="line">        Port patch-int</div><div class="line">            Interface patch-int</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-tun&#125;</div><div class="line">        Port <span class="string">"gre-1"</span></div><div class="line">            Interface <span class="string">"gre-1"</span></div><div class="line">                <span class="built_in">type</span>: gre</div><div class="line">                options: &#123;in_key=flow, local_ip=<span class="string">"10.0.0.101"</span>, out_key=flow, remote_ip=<span class="string">"10.0.0.100"</span>&#125;</div><div class="line">        Port br-tun</div><div class="line">            Interface br-tun</div><div class="line">                <span class="built_in">type</span>: internal</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，</p>
<ul>
<li>patch-int（即端口 F，端口号为1）是连接到 br-int 上的 veth pair 的端口</li>
<li>gre-1 端口（即端口 G，端口号为2）对应vm到外面的隧道。gre-1 端口是虚拟 gre 端口，当网包发送到这个端口的时候，会经过内核封包，然后从 10.0.0.101 发送到 10.0.0.100，即从本地的物理网卡（10.0.0.101）发出。</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-tun将带有 vlan tag 的 vm 跟外部通信的流量转换到对应的 gre 隧道，这上面要实现主要的转换逻辑，规则要复杂，一般通过多张表来实现。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;典型的转发规则为：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-tun</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=10970.064s, table=0, n_packets=189, n_bytes=16232, idle_age=16, priority=1,in_port=1 actions=resubmit(,1)</div><div class="line"> cookie=0x0, duration=10906.954s, table=0, n_packets=29, n_bytes=5736, idle_age=16, priority=1,in_port=2 actions=resubmit(,2)</div><div class="line"> cookie=0x0, duration=10969.922s, table=0, n_packets=3, n_bytes=230, idle_age=10962, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=10969.777s, table=1, n_packets=26, n_bytes=5266, idle_age=16, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)</div><div class="line"> cookie=0x0, duration=10969.631s, table=1, n_packets=163, n_bytes=10966, idle_age=21, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,21)</div><div class="line"> cookie=0x0, duration=688.456s, table=2, n_packets=29, n_bytes=5736, idle_age=16, priority=1,tun_id=0x1 actions=mod_vlan_vid:1,resubmit(,10)</div><div class="line"> cookie=0x0, duration=10969.488s, table=2, n_packets=0, n_bytes=0, idle_age=10969, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=10969.343s, table=3, n_packets=0, n_bytes=0, idle_age=10969, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=10969.2s, table=10, n_packets=29, n_bytes=5736, idle_age=16, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0-&gt;NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1</div><div class="line"> cookie=0x0, duration=682.603s, table=20, n_packets=26, n_bytes=5266, hard_timeout=300, idle_age=16, hard_age=16, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:32:0d:db actions=load:0-&gt;NXM_OF_VLAN_TCI[],load:0x1-&gt;NXM_NX_TUN_ID[],output:2</div><div class="line"> cookie=0x0, duration=10969.057s, table=20, n_packets=0, n_bytes=0, idle_age=10969, priority=0 actions=resubmit(,21)</div><div class="line"> cookie=0x0, duration=688.6s, table=21, n_packets=161, n_bytes=10818, idle_age=21, priority=1,dl_vlan=1 actions=strip_vlan,set_tunnel:0x1,output:2</div><div class="line"> cookie=0x0, duration=10968.912s, table=21, n_packets=2, n_bytes=148, idle_age=689, priority=0 actions=drop</div></pre></td></tr></table></figure>
<h3 id="表-0"><a href="#表-0" class="headerlink" title="表 0"></a>表 0</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，表 0 中有 3 条规则：从内部端口 1（即patch-int）来的，扔到表 1，从外部端口2（即 gre-1）来的，扔到表2。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=10970.064s, table=0, n_packets=189, n_bytes=16232, idle_age=16, priority=1,in_port=1 actions=resubmit(,1)</div><div class="line"> cookie=0x0, duration=10906.954s, table=0, n_packets=29, n_bytes=5736, idle_age=16, priority=1,in_port=2 actions=resubmit(,2)</div><div class="line"> cookie=0x0, duration=10969.922s, table=0, n_packets=3, n_bytes=230, idle_age=10962, priority=0 actions=drop</div></pre></td></tr></table></figure>
<h3 id="表-1"><a href="#表-1" class="headerlink" title="表 1"></a>表 1</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表 1 处理内部过来的网包，有 2 条规则：如果是单播（00:00:00:00:00:00/01:00:00:00:00:00），则扔到表 20；如果是多播等（01:00:00:00:00:00/01:00:00:00:00:00），则扔到表 21。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=10969.777s, table=1, n_packets=26, n_bytes=5266, idle_age=16, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)</div><div class="line"> cookie=0x0, duration=10969.631s, table=1, n_packets=163, n_bytes=10966, idle_age=21, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,21)</div></pre></td></tr></table></figure>
<h3 id="表-2"><a href="#表-2" class="headerlink" title="表 2"></a>表 2</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表 2 处理外部过来的包。有 2 条规则：如果是 tunnel 1（合法的 tunnel id）的网包，则修改其 vlan id 为1，并扔到表 10 学习记录来源；非 tunnel 1（非法的 tunnel id）的网包，则丢弃。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=688.456s, table=2, n_packets=29, n_bytes=5736, idle_age=16, priority=1,tun_id=0x1 actions=mod_vlan_vid:1,resubmit(,10)</div><div class="line"> cookie=0x0, duration=10969.488s, table=2, n_packets=0, n_bytes=0, idle_age=10969, priority=0 actions=drop</div></pre></td></tr></table></figure>
<h3 id="表-3"><a href="#表-3" class="headerlink" title="表 3"></a>表 3</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表 3 只有 1 条规则：丢弃。</p>
<h3 id="表-10"><a href="#表-10" class="headerlink" title="表 10"></a>表 10</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表 10 负责学习。有一条规则，基于 learn 行动来创建反向（内部网包从 gre 端口发出去）的规则。learn 行动并非标准的 openflow 行动，是 openvswitch 自身的扩展行动，这个行动可以根据流内容动态来修改流表内容。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这条规则首先创建了一条新的流（该流对应 vm 从 br-tun 的 gre 端口发出的规则）：其中 table=20 表示规则添加在表 20；NXM_OF_VLAN_TCI[0..11] 表示匹配包自带的v lan id；NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[] 表示 L2 目标地址需要匹配当前包的 L2 源地址；load:0-&gt;NXM_OF_VLAN_TCI[]，去掉vlan，load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[]，添加 tunnel 号为原始 tunnel 号；output:NXM_OF_IN_PORT[]，发出端口为原始包抵达的端口。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;向表 20 添加完规则后，最后将匹配的当前网包从端口 1（即 patch-int）发出。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=10969.2s, table=10, n_packets=29, n_bytes=5736, idle_age=16, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0-&gt;NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1</div></pre></td></tr></table></figure>
<h3 id="表-20"><a href="#表-20" class="headerlink" title="表 20"></a>表 20</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表 20 中有两条规则，其中第一条即表 10 中规则利用 learn 行动创建的内部向外部发包的流表项，第 2 条提交其他流到表 21。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=682.603s, table=20, n_packets=26, n_bytes=5266, hard_timeout=300, idle_age=16, hard_age=16, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:32:0d:db actions=load:0-&gt;NXM_OF_VLAN_TCI[],load:0x1-&gt;NXM_NX_TUN_ID[],output:2</div><div class="line"> cookie=0x0, duration=10969.057s, table=20, n_packets=0, n_bytes=0, idle_age=10969, priority=0 actions=resubmit(,21)</div></pre></td></tr></table></figure>
<h3 id="表21"><a href="#表21" class="headerlink" title="表21"></a>表21</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表21有2条规则，第一条是匹配所有目标vlan为1的网包，去掉vlan，然后从端口2（gre 端口）发出。第二条是丢弃。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=688.6s, table=21, n_packets=161, n_bytes=10818, idle_age=21, priority=1,dl_vlan=1 actions=strip_vlan,set_tunnel:0x1,output:2</div><div class="line"> cookie=0x0, duration=10968.912s, table=21, n_packets=2, n_bytes=148, idle_age=689, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这些规则所组成的整体转发逻辑如下图所示。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/03.png?raw=true" alt="03"></p>
<h2 id="网络节点"><a href="#网络节点" class="headerlink" title="网络节点"></a>网络节点</h2><h3 id="br-tun-1"><a href="#br-tun-1" class="headerlink" title="br-tun"></a>br-tun</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div></pre></td><td class="code"><pre><div class="line">Bridge br-tun</div><div class="line">        Port br-tun</div><div class="line">            Interface br-tun</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port patch-int</div><div class="line">            Interface patch-int</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-tun&#125;</div><div class="line">        Port <span class="string">"gre-2"</span></div><div class="line">            Interface <span class="string">"gre-2"</span></div><div class="line">                <span class="built_in">type</span>: gre</div><div class="line">                options: &#123;in_key=flow, local_ip=<span class="string">"10.0.0.100"</span>, out_key=flow, remote_ip=<span class="string">"10.0.0.101"</span>&#125;</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;Compute 节点上发往 GRE 隧道的网包最终抵达 Network 节点上的 br-tun，该网桥的规则包括：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-tun</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line">cookie=0x0, duration=19596.862s, table=0, n_packets=344, n_bytes=66762, idle_age=4, priority=1,in_port=1 actions=resubmit(,1)</div><div class="line"> cookie=0x0, duration=19537.588s, table=0, n_packets=625, n_bytes=125972, idle_age=4, priority=1,in_port=2 actions=resubmit(,2)</div><div class="line"> cookie=0x0, duration=19596.602s, table=0, n_packets=2, n_bytes=140, idle_age=19590, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=19596.343s, table=1, n_packets=323, n_bytes=65252, idle_age=4, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)</div><div class="line"> cookie=0x0, duration=19596.082s, table=1, n_packets=21, n_bytes=1510, idle_age=5027, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,21)</div><div class="line"> cookie=0x0, duration=9356.289s, table=2, n_packets=625, n_bytes=125972, idle_age=4, priority=1,tun_id=0x1 actions=mod_vlan_vid:1,resubmit(,10)</div><div class="line"> cookie=0x0, duration=19595.821s, table=2, n_packets=0, n_bytes=0, idle_age=19595, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=19595.554s, table=3, n_packets=0, n_bytes=0, idle_age=19595, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=19595.292s, table=10, n_packets=625, n_bytes=125972, idle_age=4, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0-&gt;NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1</div><div class="line"> cookie=0x0, duration=9314.338s, table=20, n_packets=323, n_bytes=65252, hard_timeout=300, idle_age=4, hard_age=3, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:cb:11:f6 actions=load:0-&gt;NXM_OF_VLAN_TCI[],load:0x1-&gt;NXM_NX_TUN_ID[],output:2</div><div class="line"> cookie=0x0, duration=19595.026s, table=20, n_packets=0, n_bytes=0, idle_age=19595, priority=0 actions=resubmit(,21)</div><div class="line"> cookie=0x0, duration=9356.592s, table=21, n_packets=9, n_bytes=586, idle_age=5027, priority=1,dl_vlan=1 actions=strip_vlan,set_tunnel:0x1,output:2</div><div class="line"> cookie=0x0, duration=19594.759s, table=21, n_packets=12, n_bytes=924, idle_age=5057, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这些规则跟 Compute 节点上 br-tun 的规则相似，完成 tunnel 跟 vlan 之间的转换。</p>
<h3 id="br-int-1"><a href="#br-int-1" class="headerlink" title="br-int"></a>br-int</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div></pre></td><td class="code"><pre><div class="line">Bridge br-int</div><div class="line">        Port <span class="string">"qr-ff19a58b-3d"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qr-ff19a58b-3d"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port br-int</div><div class="line">            Interface br-int</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port patch-tun</div><div class="line">            Interface patch-tun</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-int&#125;</div><div class="line">        Port <span class="string">"tap4385f950-8b"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"tap4385f950-8b"</span></div><div class="line">                <span class="built_in">type</span>: internal</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;该集成网桥上挂载了很多进程来提供网络服务，包括路由器、DHCP服务器等。这些进程不同的租户可能都需要，彼此的地址空间可能冲突，也可能跟物理网络的地址空间冲突，因此都运行在独立的网络名字空间中。 规则跟computer节点的br-int规则一致，表现为一个正常交换机。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-int</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=18198.244s, table=0, n_packets=849, n_bytes=164654, idle_age=43, priority=1 actions=NORMAL</div></pre></td></tr></table></figure>
<h3 id="网络名字空间"><a href="#网络名字空间" class="headerlink" title="网络名字空间"></a>网络名字空间</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 Linux 中，网络名字空间可以被认为是隔离的拥有单独网络栈（网卡、路由转发表、iptables）的环境。网络名字空间经常用来隔离网络设备和服务，只有拥有同样网络名字空间的设备，才能看到彼此。 可以用ip netns list命令来查看已经存在的名字空间。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">ip netns</div><div class="line">qdhcp-88b1609c-68e0-49ca-a658-f1edff54a264</div><div class="line">qrouter-2d214fde-293c-4d64-8062-797f80ae2d8f</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;qdhcp 开头的名字空间是 dhcp 服务器使用的，qrouter 开头的则是 router 服务使用的。 可以通过 ip netns exec namespaceid command 来在指定的网络名字空间中执行网络命令，例如</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">ip netns <span class="built_in">exec</span> qdhcp-88b1609c-68e0-49ca-a658-f1edff54a264 ip addr</div><div class="line">71: ns-f14c598d-98: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc pfifo_fast state UP qlen 1000</div><div class="line">    link/ether fa:16:3e:10:2f:03 brd ff:ff:ff:ff:ff:ff</div><div class="line">    inet 10.1.0.3/24 brd 10.1.0.255 scope global ns-f14c598d-98</div><div class="line">    inet6 fe80::f816:3eff:fe10:2f03/64 scope link</div><div class="line">       valid_lft forever preferred_lft forever</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以看到，dhcp 服务的网络名字空间中只有一个网络接口“ns-f14c598d-98”，它连接到 br-int 的 tapf14c598d-98 接口上。</p>
<h3 id="dhcp-服务"><a href="#dhcp-服务" class="headerlink" title="dhcp 服务"></a>dhcp 服务</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;dhcp 服务是通过 dnsmasq 进程（轻量级服务器，可以提供 dns、dhcp、tftp 等服务）来实现的，该进程绑定到 dhcp 名字空间中的 br-int 的接口上。可以查看相关的进程。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">ps -fe | grep 88b1609c-68e0-49ca-a658-f1edff54a264</div><div class="line">nobody   23195     1  0 Oct26 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --<span class="built_in">bind</span>-interfaces --interface=ns-f14c598d-98 --except-interface=lo --pid-file=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/host --dhcp-optsfile=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/opts --dhcp-script=/usr/bin/neutron-dhcp-agent-dnsmasq-lease-update --leasefile-ro --dhcp-range=tag0,10.1.0.0,static,120s --conf-file= --domain=openstacklocal</div><div class="line">root     23196 23195  0 Oct26 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --<span class="built_in">bind</span>-interfaces --interface=ns-f14c598d-98 --except-interface=lo --pid-file=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/host --dhcp-optsfile=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/opts --dhcp-script=/usr/bin/neutron-dhcp-agent-dnsmasq-lease-update --leasefile-ro --dhcp-range=tag0,10.1.0.0,static,120s --conf-file= --domain=openstacklocal</div></pre></td></tr></table></figure>
<h3 id="router服务"><a href="#router服务" class="headerlink" title="router服务"></a>router服务</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;首先，要理解什么是 router，router 是提供跨 subnet 的互联功能的。比如用户的内部网络中主机想要访问外部互联网的地址，就需要 router 来转发（因此，所有跟外部网络的流量都必须经过 router）。目前 router 的实现是通过 iptables 进行的。 同样的，router 服务也运行在自己的名字空间中，可以通过如下命令查看：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div></pre></td><td class="code"><pre><div class="line">ip netns <span class="built_in">exec</span> qrouter-2d214fde-293c-4d64-8062-797f80ae2d8f ip addr</div><div class="line">66: qg-d48b49e0-aa: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc pfifo_fast state UP qlen 1000</div><div class="line">    link/ether fa:16:3e:5c:a2:ac brd ff:ff:ff:ff:ff:ff</div><div class="line">    inet 172.24.4.227/28 brd 172.24.4.239 scope global qg-d48b49e0-aa</div><div class="line">    inet 172.24.4.228/32 brd 172.24.4.228 scope global qg-d48b49e0-aa</div><div class="line">    inet6 fe80::f816:3eff:fe5c:a2ac/64 scope link</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">68: qr-c2d7dd02-56: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc pfifo_fast state UP qlen 1000</div><div class="line">    link/ether fa:16:3e:ea:64:6e brd ff:ff:ff:ff:ff:ff</div><div class="line">    inet 10.1.0.1/24 brd 10.1.0.255 scope global qr-c2d7dd02-56</div><div class="line">    inet6 fe80::f816:3eff:feea:646e/64 scope link</div><div class="line">       valid_lft forever preferred_lft forever</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以看出，该名字空间中包括两个网络接口。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;第一个接口 qg-d48b49e0-aa（即K）是外部接口（qg=q gateway），将路由器的网关指向默认网关（通过 router-gateway-set 命令指定），这个接口连接到 br-ex 上的tapd48b49e0-aa（即L）。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;第二个接口 qr-c2d7dd02-56（即 N，qr=q bridge）跟 br-int 上的 tapc2d7dd02-56 口（即 M）相连，将 router 进程连接到集成网桥上。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;查看该名字空间中的路由表：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">ip netns <span class="built_in">exec</span> qrouter-2d214fde-293c-4d64-8062-797f80ae2d8f ip route</div><div class="line">172.24.4.224/28 dev qg-d48b49e0-aa  proto kernel  scope link  src 172.24.4.227</div><div class="line">10.1.0.0/24 dev qr-c2d7dd02-56  proto kernel  scope link  src 10.1.0.1</div><div class="line">default via 172.24.4.225 dev qg-d48b49e0-aa</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，第一条规则是将到 172.24.4.224/28 段的访问都从网卡 qg-d48b49e0-aa（即 K）发出。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;第二条规则是将到 10.1.0.0/24 段的访问都从网卡 qr-c2d7dd02-56（即 N）发出。 最后一条是默认路由，所有的通过 qg-d48b49e0-aa 网卡（即 K）发出。 floating ip 服务同样在路由器名字空间中实现，例如如果绑定了外部的 floating ip 172.24.4.228 到某个虚拟机 10.1.0.2，则 nat 表中规则为：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div></pre></td><td class="code"><pre><div class="line">ip netns <span class="built_in">exec</span> qrouter-2d214fde-293c-4d64-8062-797f80ae2d8f iptables -t nat -S</div><div class="line">-P PREROUTING ACCEPT</div><div class="line">-P POSTROUTING ACCEPT</div><div class="line">-P OUTPUT ACCEPT</div><div class="line">-N neutron-l3-agent-OUTPUT</div><div class="line">-N neutron-l3-agent-POSTROUTING</div><div class="line">-N neutron-l3-agent-PREROUTING</div><div class="line">-N neutron-l3-agent-float-snat</div><div class="line">-N neutron-l3-agent-snat</div><div class="line">-N neutron-postrouting-bottom</div><div class="line">-A PREROUTING -j neutron-l3-agent-PREROUTING</div><div class="line">-A POSTROUTING -j neutron-l3-agent-POSTROUTING</div><div class="line">-A POSTROUTING -j neutron-postrouting-bottom</div><div class="line">-A OUTPUT -j neutron-l3-agent-OUTPUT</div><div class="line">-A neutron-l3-agent-OUTPUT -d 172.24.4.228/32 -j DNAT --to-destination 10.1.0.2</div><div class="line">-A neutron-l3-agent-POSTROUTING ! -i qg-d48b49e0-aa ! -o qg-d48b49e0-aa -m conntrack ! --ctstate DNAT -j ACCEPT</div><div class="line">-A neutron-l3-agent-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697</div><div class="line">-A neutron-l3-agent-PREROUTING -d 172.24.4.228/32 -j DNAT --to-destination 10.1.0.2</div><div class="line">-A neutron-l3-agent-float-snat -s 10.1.0.2/32 -j SNAT --to-source 172.24.4.228</div><div class="line">-A neutron-l3-agent-snat -j neutron-l3-agent-float-snat</div><div class="line">-A neutron-l3-agent-snat -s 10.1.0.0/24 -j SNAT --to-source 172.24.4.227</div><div class="line">-A neutron-postrouting-bottom -j neutron-l3-agent-snat</div><div class="line"></div><div class="line">其中 SNAT 和 DNAT 规则完成外部 floating ip 到内部 ip 的映射：</div><div class="line">-A neutron-l3-agent-OUTPUT -d 172.24.4.228/32 -j DNAT --to-destination 10.1.0.2</div><div class="line">-A neutron-l3-agent-PREROUTING -d 172.24.4.228/32 -j DNAT --to-destination 10.1.0.2</div><div class="line">-A neutron-l3-agent-float-snat -s 10.1.0.2/32 -j SNAT --to-source 172.24.4.228</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;另外有一条 SNAT 规则把所有其他的内部IP出来的流量都映射到外部IP 172.24.4.227。这样即使在内部虚拟机没有外部IP的情况下，也可以发起对外网的访问。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">-A neutron-l3-agent-snat -s 10.1.0.0/24 -j SNAT --to-source 172.24.4.227</div></pre></td></tr></table></figure>
<h3 id="br-ex"><a href="#br-ex" class="headerlink" title="br-ex"></a>br-ex</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div></pre></td><td class="code"><pre><div class="line">Bridge br-ex</div><div class="line">        Port <span class="string">"eth1"</span></div><div class="line">            Interface <span class="string">"eth1"</span></div><div class="line">        Port br-ex</div><div class="line">            Interface br-ex</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qg-1c3627de-1b"</span></div><div class="line">            Interface <span class="string">"qg-1c3627de-1b"</span></div><div class="line">                <span class="built_in">type</span>: internal</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-ex 上直接连接到外部物理网络，一般情况下网关在物理网络中已经存在，则直接转发即可。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line"> ovs-ofctl dump-flows br-ex</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=23431.091s, table=0, n_packets=893539, n_bytes=504805376, idle_age=0, priority=0 actions=NORMAL</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;如果对外部网络的网关地址配置到了 br-ex（即br-ex作为一个网关）：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">ip addr add 172.24.4.225/28 dev br-ex</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;需要将内部虚拟机发出的流量进行 SNAT，之后发出。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">iptables -A FORWARD -d 172.24.4.224/28 -j ACCEPT</div><div class="line">iptables -A FORWARD -s 172.24.4.224/28 -j ACCEPT</div><div class="line">iptables -t nat -I POSTROUTING 1 -s 172.24.4.224/28 -j MASQUERADE</div></pre></td></tr></table></figure>
<h2 id="VLAN-模式"><a href="#VLAN-模式" class="headerlink" title="VLAN 模式"></a>VLAN 模式</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;Vlan模式下的系统架构跟GRE模式下类似，如下图所示。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;需要注意的是，在vlan模式下，vlan tag的转换需要在br-int和br-ethx两个网桥上进行相互配合。即br-int负责从int-br-ethX过来的包（带外部vlan）转换为内部vlan，而br-ethx负责从phy-br-ethx过来的包（带内部vlan）转化为外部的vlan。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/04.png?raw=true" alt="04"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;下面进行一些细节的补充讨论，以Vlan作为物理网络隔离的实现。假如要实现同一个租户下两个子网，如下图所示：</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/05.png?raw=true" alt="05"></p>
<h3 id="计算节点-1"><a href="#计算节点-1" class="headerlink" title="计算节点"></a>计算节点</h3><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/06.png?raw=true" alt="06"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;查看网桥信息，主要包括两个网桥：br-int和br-eth1：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div></pre></td><td class="code"><pre><div class="line">[root@Compute ~]<span class="comment"># ovs-vsctl show</span></div><div class="line">f758a8b8-2fd0-4a47-ab2d-c49d48304f82</div><div class="line">    Bridge <span class="string">"br-eth1"</span></div><div class="line">        Port <span class="string">"phy-br-eth1"</span></div><div class="line">            Interface <span class="string">"phy-br-eth1"</span></div><div class="line">        Port <span class="string">"br-eth1"</span></div><div class="line">            Interface <span class="string">"br-eth1"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"eth1"</span></div><div class="line">            Interface <span class="string">"eth1"</span></div><div class="line">    Bridge br-int</div><div class="line">        Port <span class="string">"qvoXXX"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qvoXXX"</span></div><div class="line">        Port <span class="string">"qvoYYY"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qvoYYY"</span></div><div class="line">        Port <span class="string">"qvoZZZ"</span></div><div class="line">             tag: 2</div><div class="line">             Interface <span class="string">"qvoZZZ"</span></div><div class="line">        Port <span class="string">"qvoWWW"</span></div><div class="line">             tag: 2</div><div class="line">             Interface <span class="string">"qvoWWW"</span></div><div class="line">        Port <span class="string">"int-br-eth1"</span></div><div class="line">            Interface <span class="string">"int-br-eth1"</span></div><div class="line">        Port br-int</div><div class="line">            Interface br-int</div><div class="line">                <span class="built_in">type</span>: internal</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;类似GRE模式下，br-int负责租户隔离，br-eth1负责跟计算节点外的网络通信。 在Vlan模式下，租户的流量隔离是通过vlan来进行的，因此此时包括两种vlan，虚拟机在Compute Node内流量带有的local vlan和在Compute Node之外物理网络上隔离不同租户的vlan。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-int和br-eth1分别对从端口int-br-eth1和phy-br-eth1上到达的网包进行vlan tag的处理。此处有两个网，分别带有两个vlan tag（内部tag1对应外部tag101，内部tag2对应外部tag102）。 其中，安全组策略仍然在qbr相关的iptables上实现。</p>
<h4 id="br-int-2"><a href="#br-int-2" class="headerlink" title="br-int"></a>br-int</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;与GRE模式不同的是，br-int完成从br-eth1上过来流量（从口int-br-eth1到达）的vlan tag转换，可能的规则为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-int</div><div class="line"> cookie=0x0, duration=100.795s, table=0, n_packets=6, n_bytes=468, idle_age=90, priority=2,in_port=3 actions=drop</div><div class="line"> cookie=0x0, duration=97.069s, table=0, n_packets=22, n_bytes=6622, idle_age=31, priority=3,in_port=3,dl_vlan=101 actions=mod_vlan_vid:1,NORMAL</div><div class="line"> cookie=0x0, duration=95.781s, table=0, n_packets=8, n_bytes=1165, idle_age=11, priority=3,in_port=3,dl_vlan=102 actions=mod_vlan_vid:2,NORMAL</div><div class="line"> cookie=0x0, duration=103.626s, table=0, n_packets=47, n_bytes=13400, idle_age=11, priority=1 actions=NORMAL</div></pre></td></tr></table></figure>
<h4 id="br-eth1"><a href="#br-eth1" class="headerlink" title="br-eth1"></a>br-eth1</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-eth1上负责从br-int上过来的流量（从口phy-br-eth1到达），实现local vlan到外部vlan的转换。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-eth0</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=73.461s, table=0, n_packets=51, n_bytes=32403, idle_age=2, hard_age=65534, priority=4,in_port=4,dl_vlan=1 actions=mod_vlan_vid:101,NORMAL</div><div class="line"> cookie=0x0, duration=83.461s, table=0, n_packets=51, n_bytes=32403, idle_age=2, hard_age=65534, priority=4,in_port=4,dl_vlan=2 actions=mod_vlan_vid:102,NORMAL</div><div class="line"> cookie=0x0, duration=651.538s, table=0, n_packets=72, n_bytes=3908, idle_age=2574, hard_age=65534, priority=2,in_port=4 actions=drop</div><div class="line"> cookie=0x0, duration=654.002s, table=0, n_packets=31733, n_bytes=6505880, idle_age=2, hard_age=65534, priority=1 actions=NORMAL</div></pre></td></tr></table></figure>
<h3 id="网络节点-1"><a href="#网络节点-1" class="headerlink" title="网络节点"></a>网络节点</h3><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/07.png?raw=true" alt="07"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;类似GRE模式下，br-eth1收到到达的网包，int-br-eth1和phy-br-eth1上分别进行vlan转换，保证到达br-int上的网包都是带有内部vlan tag，到达br-eth1上的都是带有外部vlan tag。br-ex则完成到OpenStack以外网络的连接。 查看网桥信息，包括三个网桥，br-eth1、br-int和br-ex。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div><div class="line">41</div></pre></td><td class="code"><pre><div class="line">ovs</div><div class="line">3bd78da8-d3b5-4112-a766-79506a7e2801</div><div class="line">    Bridge br-ex</div><div class="line">        Port <span class="string">"qg-VVV"</span></div><div class="line">            Interface <span class="string">"qg-VVV"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port br-ex</div><div class="line">            Interface br-ex</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"eth0"</span></div><div class="line">            Interface <span class="string">"eth0"</span></div><div class="line">    Bridge br-int</div><div class="line">        Port br-int</div><div class="line">            Interface br-int</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"int-br-eth1"</span></div><div class="line">            Interface <span class="string">"int-br-eth0"</span></div><div class="line">        Port <span class="string">"tapXXX"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"tapXXX"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">Port <span class="string">"tapWWW"</span></div><div class="line">            tag: 2</div><div class="line">            Interface <span class="string">"tapWWW"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qr-YYY"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qr-YYY"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qr-ZZZ"</span></div><div class="line">            tag: 2</div><div class="line">            Interface <span class="string">"qr-ZZZ"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">    Bridge <span class="string">"br-eth1"</span></div><div class="line">        Port <span class="string">"phy-br-eth1"</span></div><div class="line">            Interface <span class="string">"phy-br-eth1"</span></div><div class="line">        Port <span class="string">"br-eth1"</span></div><div class="line">            Interface <span class="string">"br-eth1"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"eth1"</span></div><div class="line">            Interface <span class="string">"eth1"</span></div></pre></td></tr></table></figure>
<h4 id="br-eth1-1"><a href="#br-eth1-1" class="headerlink" title="br-eth1"></a>br-eth1</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-eth1主要负责把物理网络上外部vlan转化为local vlan。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-eth1</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=144.33s, table=0, n_packets=13, n_bytes=28404, idle_age=24, hard_age=65534, priority=4,in_port=5,dl_vlan=101 actions=mod_vlan_vid:1,NORMAL</div><div class="line"> cookie=0x0, duration=144.33s, table=0, n_packets=13, n_bytes=28404, idle_age=24, hard_age=65534, priority=4,in_port=5,dl_vlan=102 actions=mod_vlan_vid:2,NORMAL</div><div class="line"> cookie=0x0, duration=608.373s, table=0, n_packets=23, n_bytes=1706, idle_age=65534, hard_age=65534, priority=2,in_port=5 actions=drop</div><div class="line"> cookie=0x0, duration=675.373s, table=0, n_packets=58, n_bytes=10625, idle_age=24, hard_age=65534, priority=1 actions=NORMAL</div></pre></td></tr></table></figure>
<h4 id="br-int-3"><a href="#br-int-3" class="headerlink" title="br-int"></a>br-int</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-int上挂载了大量的agent来提供各种网络服务，另外负责对发往br-eth1的流量，实现local vlan转化为外部vlan。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-int</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=147294.121s, table=0, n_packets=224, n_bytes=33961, idle_age=13, hard_age=65534, priority=3,in_port=4,dl_vlan=1 actions=mod_vlan_vid:101,NORMAL</div><div class="line"> cookie=0x0, duration=603538.84s, table=0, n_packets=19, n_bytes=2234, idle_age=18963, hard_age=65534, priority=2,in_port=4 actions=drop</div><div class="line"> cookie=0x0, duration=603547.134s, table=0, n_packets=31901, n_bytes=6419756, idle_age=13, hard_age=65534, priority=1 actions=NORMAL</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;dnsmasq负责提供DHCP服务，绑定到某个特定的名字空间上，每个需要DHCP服务的租户网络有自己专属隔离的DHCP服务（图中的tapXXX和tapWWW上各自监听了一个dnsmasq）。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;路由是L3 agent来实现，每个子网在br-int上有一个端口（qr-YYY和qr-ZZZ，已配置IP，分别是各自内部子网的网关），L3 agent绑定到上面。要访问外部的公共网络，需要通过L3 agent发出，而不是经过int-br-ex到phy-br-ex（实际上并没有网包从这个veth pair传输）。如果要使用外部可见的floating IP，L3 agent仍然需要通过iptables来进行NAT。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;每个L3 agent或dnsmasq都在各自独立的名字空间中，如下图所示，其中同一租户的两个子网都使用了同一个路由器。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/08.png?raw=true" alt="08"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对于子网使用不同路由器的情况，多个路由器会在自己独立的名字空间中。例如要实现两个租户的两个子网的情况，如下图所示。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/09.png?raw=true" alt="09"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这种情况下，网络节点上的名字空间如下图所示。 </p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/09.png?raw=true" alt="10"></p>
<h4 id="br-ex-1"><a href="#br-ex-1" class="headerlink" title="br-ex"></a>br-ex</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-ex要做的事情很简单，只需要正常转发即可。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">ovs-ofctl dump-flows br-ex</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=6770.969s, table=0, n_packets=5411, n_bytes=306944, idle_age=0, hard_age=65534, priority=0 actions=NORMAL</div></pre></td></tr></table></figure>
<h2 id="VXLAN-模式"><a href="#VXLAN-模式" class="headerlink" title="VXLAN 模式"></a>VXLAN 模式</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;VXLAN 模式下，网络的架构跟 GRE 模式类似，所不同的是，不同节点之间通过 VXLAN 隧道互通，即虚拟化层是采用的 VXLAN 协议。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;基本结构如下图所示。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/11.png?raw=true" alt="11"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，节点网络配置如下所示，注意数据网络接口需要 IP 地址，因为是隧道协议需要底下的三层转发支持。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;控制节点</p>
<ul>
<li>eth0: 9.186.100.77/24 作为管理网络（同时也是公共网络）。</li>
<li>eth1: 10.0.100.77/24 作为数据网络接口。</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;计算节点</p>
<ul>
<li>eth0: 9.186.100.88/24 作为管理网络（同时也是公共网络）。</li>
<li>eth1: 10.0.100.88/24 作为数据网络接口。</li>
</ul>
<h3 id="计算节点-2"><a href="#计算节点-2" class="headerlink" title="计算节点"></a>计算节点</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要包括两个网桥：集成网桥 br-int 和 隧道网桥 br-tun。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div></pre></td><td class="code"><pre><div class="line">sudo ovs-vsctl show</div><div class="line">225f3eb5-6059-4063-99c3-8666915c9c55</div><div class="line">    Bridge br-int</div><div class="line">        fail_mode: secure</div><div class="line">        Port br-int</div><div class="line">            Interface br-int</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qvoc4493802-43"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qvoc4493802-43"</span></div><div class="line">        Port patch-tun</div><div class="line">            Interface patch-tun</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-int&#125;</div><div class="line">        Port <span class="string">"qvof47c62b0-db"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qvof47c62b0-db"</span></div><div class="line">    Bridge br-tun</div><div class="line">        fail_mode: secure</div><div class="line">        Port <span class="string">"vxlan-0a00644d"</span></div><div class="line">            Interface <span class="string">"vxlan-0a00644d"</span></div><div class="line">                <span class="built_in">type</span>: vxlan</div><div class="line">                options: &#123;df_default=<span class="string">"true"</span>, in_key=flow, local_ip=<span class="string">"10.0.100.88"</span>, out_key=flow, remote_ip=<span class="string">"10.0.100.77"</span>&#125;</div><div class="line">        Port patch-int</div><div class="line">            Interface patch-int</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-tun&#125;</div><div class="line">        Port br-tun</div><div class="line">            Interface br-tun</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">    ovs_version: <span class="string">"2.0.2"</span></div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;安全网桥可以通过 brctl show 命令看到，该网桥主要用于绑定控制组的 iptables 规则，跟转发无直接关系。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">brctl show</div><div class="line">bridge name     bridge id               STP enabled     interfaces</div><div class="line">qbrf47c62b0-db          8000.56a7904c418d       no              qvbf47c62b0-db</div><div class="line">                                                        tapf47c62b0-db</div></pre></td></tr></table></figure>
<h4 id="br-int-4"><a href="#br-int-4" class="headerlink" title="br-int"></a>br-int</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;集成网桥 br-int 规则比较简单，作为一个正常的二层交换机使用。无论下面虚拟化层是哪种技术实现，集成网桥是看不到的，只知道根据 vlan 和 mac 进行转发。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;所连接接口除了从安全网桥过来的 qvo-xxx（每个虚拟机会有一个），就是一个往外的 patch-tun 接口，连接到 br-tun 网桥。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，qvo-xxx 接口上会为每个网络分配一个内部 vlan 号，比如这里是同一个网络启动了两台虚机，所以 tag 都为 1。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line">Bridge br-int</div><div class="line">        fail_mode: secure</div><div class="line">        Port br-int</div><div class="line">            Interface br-int</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qvoc4493802-43"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qvoc4493802-43"</span></div><div class="line">        Port patch-tun</div><div class="line">            Interface patch-tun</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-int&#125;</div><div class="line">        Port <span class="string">"qvof47c62b0-db"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qvof47c62b0-db"</span></div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;转发规则表 0 中是对所有包进行 NORMAL，表 23 中是所有包直接丢弃（是否后面将安全组规则在这里实现？）。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">sudo ovs-ofctl dump-flows br-int</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=52889.682s, table=0, n_packets=161, n_bytes=39290, idle_age=13, priority=1 actions=NORMAL</div><div class="line"> cookie=0x0, duration=52889.451s, table=23, n_packets=0, n_bytes=0, idle_age=52889, priority=0 actions=drop</div></pre></td></tr></table></figure>
<h4 id="br-tun-2"><a href="#br-tun-2" class="headerlink" title="br-tun"></a>br-tun</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-tun 作为虚拟化层网桥，规则就要复杂一些。 要将内部过来的网包进行合理甄别，内部带着正确 vlan tag 过来的，从正确的 tunnel 扔出去；外面带着正确 tunnel 号过来的，要改到对应的内部 vlan tag 扔到里面。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div></pre></td><td class="code"><pre><div class="line">Bridge br-tun</div><div class="line">        fail_mode: secure</div><div class="line">        Port <span class="string">"vxlan-0a00644d"</span></div><div class="line">            Interface <span class="string">"vxlan-0a00644d"</span></div><div class="line">                <span class="built_in">type</span>: vxlan</div><div class="line">                options: &#123;df_default=<span class="string">"true"</span>, in_key=flow, local_ip=<span class="string">"10.0.100.88"</span>, out_key=flow, remote_ip=<span class="string">"10.0.100.77"</span>&#125;</div><div class="line">        Port patch-int</div><div class="line">            Interface patch-int</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-tun&#125;</div><div class="line">        Port br-tun</div><div class="line">            Interface br-tun</div><div class="line">                <span class="built_in">type</span>: internal</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，端口 br-tun 是内部端口，vxlan-0a00644d 这样的端口是向其它节点发包时候的 VXLAN 隧道端点，patch-int 端口通过一条管道连接到 br-int 上的 patch-tun 端口。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;正常情况下，虚拟机的流量经过 br-int 转发，经过 patch-tun 端口，抵达 patch-int 端口，从而到达 br-tun 网桥，该网桥根据自身规则将合适的网包经过 VXLAN 隧道送出去。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line">sudo ovs-ofctl dump-flows br-tun</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=329.194s, table=0, n_packets=31, n_bytes=2906, idle_age=29, priority=1,in_port=1 actions=resubmit(,2)</div><div class="line"> cookie=0x0, duration=325.847s, table=0, n_packets=14, n_bytes=1591, idle_age=33, priority=1,in_port=2 actions=resubmit(,4)</div><div class="line"> cookie=0x0, duration=328.954s, table=0, n_packets=6, n_bytes=480, idle_age=321, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=328.712s, table=2, n_packets=9, n_bytes=694, idle_age=33, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)</div><div class="line"> cookie=0x0, duration=328.465s, table=2, n_packets=22, n_bytes=2212, idle_age=29, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)</div><div class="line"> cookie=0x0, duration=328.223s, table=3, n_packets=0, n_bytes=0, idle_age=328, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=50.703s, table=4, n_packets=12, n_bytes=1451, idle_age=33, priority=1,tun_id=0x3e9 actions=mod_vlan_vid:1,resubmit(,10)</div><div class="line"> cookie=0x0, duration=327.979s, table=4, n_packets=2, n_bytes=140, idle_age=94, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=327.742s, table=10, n_packets=12, n_bytes=1451, idle_age=33, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0-&gt;NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1</div><div class="line"> cookie=0x0, duration=38.551s, table=20, n_packets=9, n_bytes=694, hard_timeout=300, idle_age=33, hard_age=33, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:83:95:fa actions=load:0-&gt;NXM_OF_VLAN_TCI[],load:0x3e9-&gt;NXM_NX_TUN_ID[],output:2</div><div class="line"> cookie=0x0, duration=327.504s, table=20, n_packets=0, n_bytes=0, idle_age=327, priority=0 actions=resubmit(,22)</div><div class="line"> cookie=0x0, duration=50.94s, table=22, n_packets=11, n_bytes=1334, idle_age=29, dl_vlan=1 actions=strip_vlan,set_tunnel:0x3e9,output:2</div><div class="line"> cookie=0x0, duration=327.261s, table=22, n_packets=10, n_bytes=808, idle_age=51, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这些规则组成如下图所示的转发逻辑。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/12.png?raw=true" alt="12"></p>
<h4 id="表-0-1"><a href="#表-0-1" class="headerlink" title="表 0"></a>表 0</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;先看 table0 中的规则</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=329.194s, table=0, n_packets=31, n_bytes=2906, idle_age=29, priority=1,in_port=1 actions=resubmit(,2)</div><div class="line"> cookie=0x0, duration=325.847s, table=0, n_packets=14, n_bytes=1591, idle_age=33, priority=1,in_port=2 actions=resubmit(,4)</div><div class="line"> cookie=0x0, duration=328.954s, table=0, n_packets=6, n_bytes=480, idle_age=321, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;从 1 端口（patch-int）进来的网包，扔给表 2 处理，从 2 端口（vxlan-0a00644d）进来的网包，扔给表 4 处理。即一个处理来自内部 vm 的，一个处理来自外面的 vxlan 隧道的。</p>
<h4 id="表-2-1"><a href="#表-2-1" class="headerlink" title="表 2"></a>表 2</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对于内部包，表 2 中规则为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=53316.397s, table=2, n_packets=0, n_bytes=0, idle_age=53316, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)</div><div class="line"> cookie=0x0, duration=53316.162s, table=2, n_packets=161, n_bytes=39562, idle_age=422, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;即里面过来的单播包，扔给表 20 处理；多播和广播包，扔给表 22 处理。</p>
<h4 id="表-3-1"><a href="#表-3-1" class="headerlink" title="表 3"></a>表 3</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;丢弃所有包。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=328.223s, table=3, n_packets=0, n_bytes=0, idle_age=328, priority=0 actions=drop</div></pre></td></tr></table></figure>
<h4 id="表-4"><a href="#表-4" class="headerlink" title="表 4"></a>表 4</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对于外部来的数据，表 4 中规则为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=50.703s, table=4, n_packets=12, n_bytes=1451, idle_age=33, priority=1,tun_id=0x3e9 actions=mod_vlan_vid:1,resubmit(,10)</div><div class="line"> cookie=0x0, duration=327.979s, table=4, n_packets=2, n_bytes=140, idle_age=94, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;匹配给定的 tunnel 号，添加对应的 vlan 号，扔给表 10 去学习一下后扔到 br-int 网桥。</p>
<h4 id="表-10-1"><a href="#表-10-1" class="headerlink" title="表 10"></a>表 10</h4><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=327.742s, table=10, n_packets=12, n_bytes=1451, idle_age=33, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0-&gt;NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要作用是学习外部（从 tunnel）进来的包，往表 20 中添加对返程包的正常转发规则，并且从 patch-int 扔给 br-int。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;使用了 openvswitch 的 learn 动作。该动作能根据处理的流来动态修改其它表中的规则。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;具体来看 learn 规则。</p>
<ul>
<li>table=20 说明是修改表 20 中的规则，后面是添加的规则内容；</li>
<li>NXM_OF_VLAN_TCI[0..11]，匹配跟当前流同样的 VLAN 头，其中 NXM 是 Nicira Extensible Match 的缩写；</li>
<li>NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[]，包的目的 mac 跟当前流的源 mac 匹配；</li>
<li>load:0-&gt;NXM_OF_VLAN_TCI[]，将 vlan 号改为 0；</li>
<li>load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[]，将 tunnel 号修改为当前的 tunnel 号；</li>
<li>output:NXM_OF_IN_PORT[]，从当前入口发出。</li>
</ul>
<h4 id="表-20-1"><a href="#表-20-1" class="headerlink" title="表 20"></a>表 20</h4><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=38.551s, table=20, n_packets=9, n_bytes=694, hard_timeout=300, idle_age=33, hard_age=33, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:83:95:fa actions=load:0-&gt;NXM_OF_VLAN_TCI[],load:0x3e9-&gt;NXM_NX_TUN_ID[],output:2</div><div class="line"> cookie=0x0, duration=327.504s, table=20, n_packets=0, n_bytes=0, idle_age=327, priority=0 actions=resubmit(,22)</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，第一条规则就是表 10 学习来的结果。对于 vlan 号为 1，目标 mac 是 fa:16:3e:83:95:fa（之前，我们从虚拟机内 ping 10.0.0.1，这个 mac 作为源 mac 从 tunnel 来过）的网包，去掉 vlan 号，添加当时的 vxlan 号，并从 tunnel 发出。<br>对于没学习到规则的网包，则扔给表 22 处理。</p>
<h4 id="表-22"><a href="#表-22" class="headerlink" title="表 22"></a>表 22</h4><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=50.94s, table=22, n_packets=11, n_bytes=1334, idle_age=29, dl_vlan=1 actions=strip_vlan,set_tunnel:0x3e9,output:2</div><div class="line"> cookie=0x0, duration=327.261s, table=22, n_packets=10, n_bytes=808, idle_age=51, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表 22 检查如果 vlan 号正确，则去掉 vlan 头后从 tunnel 扔出去。</p>
<h2 id="网络节点-2"><a href="#网络节点-2" class="headerlink" title="网络节点"></a>网络节点</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;网络节点担负着进行网络服务的任务，包括DHCP、路由和高级网络服务等。一般包括三个网桥：br-tun、br-int 和 br-ex。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div></pre></td><td class="code"><pre><div class="line">sudo ovs-vsctl show</div><div class="line">49761e8e-031f-4a60-b838-28bb82aac7b7</div><div class="line">    Bridge br-int</div><div class="line">        fail_mode: secure</div><div class="line">        Port br-int</div><div class="line">            Interface br-int</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qr-694450d6-f6"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qr-694450d6-f6"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"tap13685e28-b0"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"tap13685e28-b0"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port patch-tun</div><div class="line">            Interface patch-tun</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-int&#125;</div><div class="line">    Bridge br-ex</div><div class="line">        Port br-ex</div><div class="line">            Interface br-ex</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qg-e76de35e-90"</span></div><div class="line">            Interface <span class="string">"qg-e76de35e-90"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">    Bridge br-tun</div><div class="line">        fail_mode: secure</div><div class="line">        Port br-tun</div><div class="line">            Interface br-tun</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"vxlan-0a006458"</span></div><div class="line">            Interface <span class="string">"vxlan-0a006458"</span></div><div class="line">                <span class="built_in">type</span>: vxlan</div><div class="line">                options: &#123;df_default=<span class="string">"true"</span>, in_key=flow, local_ip=<span class="string">"10.0.100.77"</span>, out_key=flow, remote_ip=<span class="string">"10.0.100.88"</span>&#125;</div><div class="line">        Port patch-int</div><div class="line">            Interface patch-int</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-tun&#125;</div><div class="line">    ovs_version: <span class="string">"2.0.2"</span></div></pre></td></tr></table></figure>
<h3 id="br-tun-3"><a href="#br-tun-3" class="headerlink" title="br-tun"></a>br-tun</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;跟计算节点类似，br-tun 作为虚拟化层网桥。 要将内部过来的网包进行合理甄别，内部带着正确 vlan tag 过来的，从正确的 tunnel 扔出去；外面带着正确 tunnel 号过来的，要改到对应的内部 vlan tag 扔到里面。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;包括两个接口，跟其它接点形成 tunnel 的 vxlan-xxx 端口，以及跟 br-int 互连的 patch-int 端口。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div></pre></td><td class="code"><pre><div class="line">Bridge br-tun</div><div class="line">        fail_mode: secure</div><div class="line">        Port br-tun</div><div class="line">            Interface br-tun</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"vxlan-0a006458"</span></div><div class="line">            Interface <span class="string">"vxlan-0a006458"</span></div><div class="line">                <span class="built_in">type</span>: vxlan</div><div class="line">                options: &#123;df_default=<span class="string">"true"</span>, in_key=flow, local_ip=<span class="string">"10.0.100.77"</span>, out_key=flow, remote_ip=<span class="string">"10.0.100.88"</span>&#125;</div><div class="line">        Port patch-int</div><div class="line">            Interface patch-int</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-tun&#125;</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，端口 br-tun 是内部端口，vxlan-0a00644d 这样的端口是向其它节点发包时候的 VXLAN 隧道端点，patch-int 端口通过一条管道连接到 br-int 上的 patch-tun 端口。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;查看 br-tun 上的转发规则。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line">sudo ovs-ofctl dump-flows br-tun</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=422.153s, table=0, n_packets=1073, n_bytes=107845, idle_age=96, priority=1,in_port=1 actions=resubmit(,2)</div><div class="line"> cookie=0x0, duration=185.009s, table=0, n_packets=1076, n_bytes=102922, idle_age=96, priority=1,in_port=2 actions=resubmit(,4)</div><div class="line"> cookie=0x0, duration=421.853s, table=0, n_packets=6, n_bytes=480, idle_age=62414, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=421.552s, table=2, n_packets=1047, n_bytes=105725, idle_age=96, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)</div><div class="line"> cookie=0x0, duration=421.252s, table=2, n_packets=26, n_bytes=2120, idle_age=61953, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)</div><div class="line"> cookie=0x0, duration=420.939s, table=3, n_packets=0, n_bytes=0, idle_age=62420, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=394.249s, table=4, n_packets=1076, n_bytes=102922, idle_age=96, priority=1,tun_id=0x3e9 actions=mod_vlan_vid:1,resubmit(,10)</div><div class="line"> cookie=0x0, duration=420.628s, table=4, n_packets=0, n_bytes=0, idle_age=62420, priority=0 actions=drop</div><div class="line"> cookie=0x0, duration=420.304s, table=10, n_packets=1076, n_bytes=102922, idle_age=96, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0-&gt;NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1</div><div class="line"> cookie=0x0, duration=1314.149s, table=20, n_packets=1006, n_bytes=101338, hard_timeout=300, idle_age=96, hard_age=95, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:52:7a:f2 actions=load:0-&gt;NXM_OF_VLAN_TCI[],load:0x3e9-&gt;NXM_NX_TUN_ID[],output:2</div><div class="line"> cookie=0x0, duration=419.977s, table=20, n_packets=0, n_bytes=0, idle_age=62419, priority=0 actions=resubmit(,22)</div><div class="line"> cookie=0x0, duration=184.683s, table=22, n_packets=3, n_bytes=230, idle_age=61953, dl_vlan=1 actions=strip_vlan,set_tunnel:0x3e9,output:2</div><div class="line"> cookie=0x0, duration=419.668s, table=22, n_packets=23, n_bytes=1890, idle_age=61961, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这些规则跟计算节点上的 br-tun 网桥规则类似，组成如下图所示的转发逻辑。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/13.png?raw=true" alt="13"></p>
<h3 id="表-0-2"><a href="#表-0-2" class="headerlink" title="表 0"></a>表 0</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;先看 table0 中的规则</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=422.153s, table=0, n_packets=1073, n_bytes=107845, idle_age=96, priority=1,in_port=1 actions=resubmit(,2)</div><div class="line">cookie=0x0, duration=185.009s, table=0, n_packets=1076, n_bytes=102922, idle_age=96, priority=1,in_port=2 actions=resubmit(,4)</div><div class="line">cookie=0x0, duration=421.853s, table=0, n_packets=6, n_bytes=480, idle_age=62414, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;从 1 端口（patch-int）进来的网包，扔给表 2 处理，从 2 端口（vxlan-0a00644d）进来的网包，扔给表 4 处理。即一个处理来自内部 br-int 的（这上面挂载着所有的网络服务，包括路由、DHCP 等），一个处理来自外面的 VXLAN 隧道的。</p>
<h3 id="表-2-2"><a href="#表-2-2" class="headerlink" title="表 2"></a>表 2</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对于内部包，表 2 中规则为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=421.552s, table=2, n_packets=1047, n_bytes=105725, idle_age=96, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)</div><div class="line">cookie=0x0, duration=421.252s, table=2, n_packets=26, n_bytes=2120, idle_age=61953, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;即里面过来的单播包，扔给表 20 处理；多播和广播包，扔给表 22 处理。</p>
<h3 id="表-3-2"><a href="#表-3-2" class="headerlink" title="表 3"></a>表 3</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;丢弃所有包。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=420.939s, table=3, n_packets=0, n_bytes=0, idle_age=62420, priority=0 actions=drop</div></pre></td></tr></table></figure>
<h3 id="表-4-1"><a href="#表-4-1" class="headerlink" title="表 4"></a>表 4</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对于外部来的数据，表 4 中规则为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=394.249s, table=4, n_packets=1076, n_bytes=102922, idle_age=96, priority=1,tun_id=0x3e9 actions=mod_vlan_vid:1,resubmit(,10)</div><div class="line">cookie=0x0, duration=420.628s, table=4, n_packets=0, n_bytes=0, idle_age=62420, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;匹配给定的 tunnel 号，添加对应的 vlan 号，扔给表 10 去学习一下后扔到 br-int 网桥。</p>
<h3 id="表-10-2"><a href="#表-10-2" class="headerlink" title="表 10"></a>表 10</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=420.304s, table=10, n_packets=1076, n_bytes=102922, idle_age=96, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0-&gt;NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要作用是学习外部（从 tunnel）进来的包，往表 20 中添加对返程包的正常转发规则，并且从 patch-int 扔给 br-int。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;使用了 openvswitch 的 learn 动作。该动作能根据处理的流来动态修改其它表中的规则。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;具体来看 learn 规则。</p>
<ul>
<li>table=20 说明是修改表 20 中的规则，后面是添加的规则内容；</li>
<li>NXM_OF_VLAN_TCI[0..11]，匹配跟当前流同样的 VLAN 头，其中 NXM 是 Nicira Extensible Match 的缩写；</li>
<li>NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[]，包的目的 mac 跟当前流的源 mac 匹配；</li>
<li>load:0-&gt;NXM_OF_VLAN_TCI[]，将 vlan 号改为 0；</li>
<li>load:NXM_NX_TUN_ID[]-&gt;NXM_NX_TUN_ID[]，将 tunnel 号修改为当前的 tunnel 号；</li>
<li>output:NXM_OF_IN_PORT[]，从当前入口发出。</li>
</ul>
<h3 id="表-20-2"><a href="#表-20-2" class="headerlink" title="表 20"></a>表 20</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=1314.149s, table=20, n_packets=1006, n_bytes=101338, hard_timeout=300, idle_age=96, hard_age=95, priority=1,vlan_tci=0x0001/0x0fff,dl_dst=fa:16:3e:52:7a:f2 actions=load:0-&gt;NXM_OF_VLAN_TCI[],load:0x3e9-&gt;NXM_NX_TUN_ID[],output:2</div><div class="line"> cookie=0x0, duration=419.977s, table=20, n_packets=0, n_bytes=0, idle_age=62419, priority=0 actions=resubmit(,22)</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，第一条规则就是表 10 学习来的结果。对于 vlan 号为 1，目标 mac 是 fa:16:3e:83:95:fa（之前，我们从虚拟机内 ping 10.0.0.1，这个 mac 作为源 mac 从 tunnel 来过）的网包，去掉 vlan 号，添加当时的 vxlan 号，并从 tunnel 发出。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对于没学习到规则的网包，则扔给表 22 处理。</p>
<h3 id="表-22-1"><a href="#表-22-1" class="headerlink" title="表 22"></a>表 22</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">cookie=0x0, duration=184.683s, table=22, n_packets=3, n_bytes=230, idle_age=61953, dl_vlan=1 actions=strip_vlan,set_tunnel:0x3e9,output:2</div><div class="line">cookie=0x0, duration=419.668s, table=22, n_packets=23, n_bytes=1890, idle_age=61961, priority=0 actions=drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表 22 检查如果 vlan 号正确，则去掉 vlan 头后从 tunnel 扔出去。</p>
<h3 id="br-int-5"><a href="#br-int-5" class="headerlink" title="br-int"></a>br-int</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;集成网桥 br-int 规则比较简单，作为一个正常的二层交换机使用。无论下面虚拟化层是哪种技术实现，集成网桥是看不到的，只知道根据 vlan 和 mac 进行转发。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;所连接接口包括：</p>
<ul>
<li>tap-xxx，连接到网络 DHCP 服务的命名空间；</li>
<li>qr-xxx，连接到路由服务的命名空间；</li>
<li>往外的 patch-tun 接口，连接到 br-tun 网桥。</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中网络服务接口上会绑定内部 vlan 号，每个号对应一个网络。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div></pre></td><td class="code"><pre><div class="line">Bridge br-int</div><div class="line">        fail_mode: secure</div><div class="line">        Port br-int</div><div class="line">            Interface br-int</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qr-694450d6-f6"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"qr-694450d6-f6"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"tap13685e28-b0"</span></div><div class="line">            tag: 1</div><div class="line">            Interface <span class="string">"tap13685e28-b0"</span></div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port patch-tun</div><div class="line">            Interface patch-tun</div><div class="line">                <span class="built_in">type</span>: patch</div><div class="line">                options: &#123;peer=patch-int&#125;</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;转发规则表 0 中是对所有包进行 NORMAL，表 23 中是所有包直接丢弃（是否后面将安全组规则在这里实现？）。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">sudo ovs-ofctl dump-flows br-int</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=52889.682s, table=0, n_packets=161, n_bytes=39290, idle_age=13, priority=1 actions=NORMAL</div><div class="line"> cookie=0x0, duration=52889.451s, table=23, n_packets=0, n_bytes=0, idle_age=52889, priority=0 actions=drop</div></pre></td></tr></table></figure>
<h3 id="br-ex-2"><a href="#br-ex-2" class="headerlink" title="br-ex"></a>br-ex</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;核心接口有两个。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;一个是挂载的物理接口上，如 eth0，网包将从这个接口发送到外部网络上。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;另外一个是 qg-xxx 这样的接口，是连接到 router 服务的网络名字空间中，里面绑定一个路由器的外部 IP，作为 nAT 时候的地址，另外，网络中的 floating IP 也放在这个网络名字空间中。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div></pre></td><td class="code"><pre><div class="line">Bridge br-ex</div><div class="line">        Port <span class="string">"eth0"</span></div><div class="line">            Interface <span class="string">"eth0"</span></div><div class="line">        Port br-ex</div><div class="line">            Interface br-ex</div><div class="line">                <span class="built_in">type</span>: internal</div><div class="line">        Port <span class="string">"qg-e76de35e-90"</span></div><div class="line">            Interface <span class="string">"qg-e76de35e-90"</span></div><div class="line">                <span class="built_in">type</span>: internal</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;网桥的规则也很简单，作为一个正常的二层转发设备即可。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">sudo ovs-ofctl dump-flows br-ex</div><div class="line">NXST_FLOW reply (xid=0x4):</div><div class="line"> cookie=0x0, duration=75072.257s, table=0, n_packets=352212, n_bytes=85641148, idle_age=0, hard_age=65534, priority=0 actions=NORMAL</div></pre></td></tr></table></figure>
<h2 id="网络名字空间-1"><a href="#网络名字空间-1" class="headerlink" title="网络名字空间"></a>网络名字空间</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 Linux 中，网络名字空间可以被认为是隔离的拥有单独网络栈（网卡、路由转发表、iptables）的环境。网络名字空间经常用来隔离网络设备和服务，只有拥有同样网络名字空间的设备，才能看到彼此。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以用ip netns list命令来查看已经存在的名字空间。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">ip net</div><div class="line">qdhcp-ea3928dc-b1fd-4a1a-940e-82b8c55214e6</div><div class="line">qrouter-40fff075-d3a2-477b-942c-6b1adb42e35e</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;qdhcp开头的名字空间是dhcp服务器使用的，qrouter开头的则是router服务使用的。 可以通过 ip netns exec namespaceid command 来在指定的网络名字空间中执行网络命令，例如</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">ip netns <span class="built_in">exec</span> qdhcp-88b1609c-68e0-49ca-a658-f1edff54a264 ip addr</div><div class="line">71: ns-f14c598d-98: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc pfifo_fast state UP qlen 1000</div><div class="line">    link/ether fa:16:3e:10:2f:03 brd ff:ff:ff:ff:ff:ff</div><div class="line">    inet 10.1.0.3/24 brd 10.1.0.255 scope global ns-f14c598d-98</div><div class="line">    inet6 fe80::f816:3eff:fe10:2f03/64 scope link</div><div class="line">       valid_lft forever preferred_lft forever</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以看到，dhcp服务的网络名字空间中只有一个网络接口“ns-f14c598d-98”，它连接到br-int的tapf14c598d-98接口上。</p>
<h3 id="DHCP-服务"><a href="#DHCP-服务" class="headerlink" title="DHCP 服务"></a>DHCP 服务</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;dhcp服务是通过dnsmasq进程（轻量级服务器，可以提供dns、dhcp、tftp等服务）来实现的，该进程绑定到dhcp名字空间中的br-int的接口上。可以查看相关的进程。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">ps -fe | grep 88b1609c-68e0-49ca-a658-f1edff54a264</div><div class="line">nobody   23195     1  0 Oct26 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --<span class="built_in">bind</span>-interfaces --interface=ns-f14c598d-98 --except-interface=lo --pid-file=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/host --dhcp-optsfile=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/opts --dhcp-script=/usr/bin/neutron-dhcp-agent-dnsmasq-lease-update --leasefile-ro --dhcp-range=tag0,10.1.0.0,static,120s --conf-file= --domain=openstacklocal</div><div class="line">root     23196 23195  0 Oct26 ?        00:00:00 dnsmasq --no-hosts --no-resolv --strict-order --<span class="built_in">bind</span>-interfaces --interface=ns-f14c598d-98 --except-interface=lo --pid-file=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/pid --dhcp-hostsfile=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/host --dhcp-optsfile=/var/lib/neutron/dhcp/88b1609c-68e0-49ca-a658-f1edff54a264/opts --dhcp-script=/usr/bin/neutron-dhcp-agent-dnsmasq-lease-update --leasefile-ro --dhcp-range=tag0,10.1.0.0,static,120s --conf-file= --domain=openstacklocal</div></pre></td></tr></table></figure>
<h3 id="路由服务"><a href="#路由服务" class="headerlink" title="路由服务"></a>路由服务</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;首先，要理解什么是 router，router是提供跨 subnet 的互联功能的。比如用户的内部网络中主机想要访问外部互联网的地址，就需要router来转发（因此，所有跟外部网络的流量都必须经过router）。目前router的实现是通过iptables进行的。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;同样的，router服务也运行在自己的名字空间中，可以通过如下命令查看：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div></pre></td><td class="code"><pre><div class="line">sudo ip net <span class="built_in">exec</span> qrouter-40fff075-d3a2-477b-942c-6b1adb42e35e ip addr</div><div class="line">1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default</div><div class="line">    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00</div><div class="line">    inet 127.0.0.1/8 scope host lo</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">    inet6 ::1/128 scope host</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">49: qr-694450d6-f6: &lt;BROADCAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UNKNOWN group default</div><div class="line">    link/ether fa:16:3e:5d:18:10 brd ff:ff:ff:ff:ff:ff</div><div class="line">    inet 10.0.0.1/24 brd 10.0.0.255 scope global qr-694450d6-f6</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">    inet6 fe80::f816:3eff:fe5d:1810/64 scope link</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">50: qg-e76de35e-90: &lt;BROADCAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UNKNOWN group default</div><div class="line">    link/ether fa:16:3e:70:24:92 brd ff:ff:ff:ff:ff:ff</div><div class="line">    inet 9.186.100.2/24 brd 9.186.100.255 scope global qg-e76de35e-90</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">    inet 9.186.100.129/32 brd 9.186.100.129 scope global qg-e76de35e-90</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">    inet6 fe80::f816:3eff:fe70:2492/64 scope link</div><div class="line">       valid_lft forever preferred_lft forever</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以看出，该名字空间中包括两个网络接口。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;第一个接口 qr-694450d6-f6（10.0.0.1）跟 br-int 上的接口相连。即任何从 br-int 来的找 10.0.0.1 （租户的私有网段）的网包都会到达这个接口。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;第一个接口 qg-e76de35e-90 连接到 br-ex 上的接口，即任何从外部来的网包，询问 9.186.100.2（默认的静态 NAT 外部地址）或 9.186.100.129（租户申请的 floating IP 地址），都会到达这个接口。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;查看该名字空间中的路由表：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">sudo ip net <span class="built_in">exec</span> qrouter-40fff075-d3a2-477b-942c-6b1adb42e35e ip route</div><div class="line">default via 9.186.100.1 dev qg-e76de35e-90</div><div class="line">9.186.100.0/24 dev qg-e76de35e-90  proto kernel  scope link  src 9.186.100.2</div><div class="line">10.0.0.0/24 dev qr-694450d6-f6  proto kernel  scope link  src 10.0.0.1</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;默认情况，以及访问外部网络的时候，休会从 qg-xxx 接口发出，经过 br-ex 发布到外网。<br>访问租户内网的时候，会从 qr-xxx 接口发出，发给 br-int。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div></pre></td><td class="code"><pre><div class="line">sudo ip net <span class="built_in">exec</span> qrouter-40fff075-d3a2-477b-942c-6b1adb42e35e iptables -t nat -S</div><div class="line">-P PREROUTING ACCEPT</div><div class="line">-P INPUT ACCEPT</div><div class="line">-P OUTPUT ACCEPT</div><div class="line">-P POSTROUTING ACCEPT</div><div class="line">-N neutron-postrouting-bottom</div><div class="line">-N neutron-vpn-agen-OUTPUT</div><div class="line">-N neutron-vpn-agen-POSTROUTING</div><div class="line">-N neutron-vpn-agen-PREROUTING</div><div class="line">-N neutron-vpn-agen-float-snat</div><div class="line">-N neutron-vpn-agen-snat</div><div class="line">-A PREROUTING -j neutron-vpn-agen-PREROUTING</div><div class="line">-A OUTPUT -j neutron-vpn-agen-OUTPUT</div><div class="line">-A POSTROUTING -j neutron-vpn-agen-POSTROUTING</div><div class="line">-A POSTROUTING -j neutron-postrouting-bottom</div><div class="line">-A neutron-postrouting-bottom -j neutron-vpn-agen-snat</div><div class="line">-A neutron-vpn-agen-OUTPUT -d 9.186.100.129/32 -j DNAT --to-destination 10.0.0.2</div><div class="line">-A neutron-vpn-agen-POSTROUTING ! -i qg-e76de35e-90 ! -o qg-e76de35e-90 -m conntrack ! --ctstate DNAT -j ACCEPT</div><div class="line">-A neutron-vpn-agen-PREROUTING -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 9697</div><div class="line">-A neutron-vpn-agen-PREROUTING -d 9.186.100.129/32 -j DNAT --to-destination 10.0.0.2</div><div class="line">-A neutron-vpn-agen-float-snat -s 10.0.0.2/32 -j SNAT --to-source 9.186.100.129</div><div class="line">-A neutron-vpn-agen-snat -j neutron-vpn-agen-float-snat</div><div class="line">-A neutron-vpn-agen-snat -s 10.0.0.0/24 -j SNAT --to-source 9.186.100.2</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中SNAT和DNAT规则完成外部 floating ip （9.186.100.129）到内部 ip（10.0.0.2） 的映射：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">-A neutron-vpn-agen-OUTPUT -d 9.186.100.129/32 -j DNAT --to-destination 10.0.0.2</div><div class="line">-A neutron-vpn-agen-PREROUTING -d 9.186.100.129/32 -j DNAT --to-destination 10.0.0.2</div><div class="line">-A neutron-vpn-agen-float-snat -s 10.0.0.2/32 -j SNAT --to-source 9.186.100.129</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;另外有一条SNAT规则把所有其他的内部IP出来的流量都映射到外部IP 9.186.100.2。这样即使在内部虚拟机没有外部IP的情况下，也可以发起对外网的访问。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">-A neutron-vpn-agen-snat -s 10.0.0.0/24 -j SNAT --to-source 9.186.100.2</div></pre></td></tr></table></figure>
<h3 id="安全组"><a href="#安全组" class="headerlink" title="安全组"></a>安全组</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;Security group通过Linux IPtables来实现，为此，在Compute节点上引入了qbr*这样的Linux传统bridge（iptables规则目前无法加载到直接挂在到ovs的tap设备上）。 首先在Control节点上用neutron port-list命令列出虚拟机的端口id，例如：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div></pre></td><td class="code"><pre><div class="line">neutron port-list</div><div class="line">+--------------------------------------+------+-------------------+----------------------------------------------------------------------------------------+</div><div class="line">| id                                 | name | mac_address       | fixed_ips                                                                  |</div><div class="line">+--------------------------------------+------+-------------------+----------------------------------------------------------------------------------------+</div><div class="line">| 2a169bb4-4d8b-4c67-802c-a24bdafe1312 |      | fa:16:3e:2f:e9:72 | &#123;<span class="string">"subnet_id"</span>: <span class="string">"a2456a2c-5eea-416d-8757-d10bc0aa2aaa"</span>, <span class="string">"ip_address"</span>: <span class="string">"192.168.0.1"</span>&#125;     |</div><div class="line">| 583c7038-d341-41ec-a0d1-0cd2c33866ca |      | fa:16:3e:9c:dc:3a | &#123;<span class="string">"subnet_id"</span>: <span class="string">"a2456a2c-5eea-416d-8757-d10bc0aa2aaa"</span>, <span class="string">"ip_address"</span>: <span class="string">"192.168.0.2"</span>&#125;     |</div><div class="line">| 9b2db4ac-3145-401c-8dc6-486ca6e303b6 |      | fa:16:3e:4e:f1:b5 | &#123;<span class="string">"subnet_id"</span>: <span class="string">"ea4ed31b-e05a-4735-8c3f-9b430e656b64"</span>, <span class="string">"ip_address"</span>: <span class="string">"192.168.122.200"</span>&#125; |</div><div class="line">| c5a7d51b-9934-40bd-befa-adff840462d2 |      | fa:16:3e:21:1d:00 | &#123;<span class="string">"subnet_id"</span>: <span class="string">"ea4ed31b-e05a-4735-8c3f-9b430e656b64"</span>, <span class="string">"ip_address"</span>: <span class="string">"192.168.122.201"</span>&#125; |</div><div class="line">| db2f5a49-7c0d-45dd-acad-908931f9a654 |      | fa:16:3e:17:5c:36 | &#123;<span class="string">"subnet_id"</span>: <span class="string">"a2456a2c-5eea-416d-8757-d10bc0aa2aaa"</span>, <span class="string">"ip_address"</span>: <span class="string">"192.168.0.3"</span>&#125;     |</div><div class="line">+--------------------------------------+------+-------------------+----------------------------------------------------------------------------------------+</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中id的前10位数字被用作虚机对外连接的qbr（同时也是tap口）的id。i或o加上前9位数字被用作安全组chain的id。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;所有的规则默认都在Compute节点上的filter表（默认表）中实现，分别来查看filter表的INPUT、OUTPUT、FORWARD三条链上的规则。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在Compute节点上，可以用 iptables –line-numbers -vnL [CHAIN] 来获得filter表（可以指定某个链上的）规则。</p>
<h3 id="INPUT"><a href="#INPUT" class="headerlink" title="INPUT"></a>INPUT</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL INPUT</div><div class="line">Chain INPUT (policy ACCEPT 0 packets, 0 bytes)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1     360K   56M neutron-openvswi-INPUT  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">2    10583 2146K ACCEPT     tcp  --  *      *       192.168.122.100      0.0.0.0/0           multiport dports 5666 /* 001 nagios-nrpe incoming 192.168.122.100 */</div><div class="line">3      846 50966 ACCEPT     tcp  --  *      *       192.168.122.100      0.0.0.0/0           multiport dports 5900:5999 /* 001 nova compute incoming 192.168.122.100 */</div><div class="line">4    1033K  894M ACCEPT     all  --  *      *       0.0.0.0/0            0.0.0.0/0           state RELATED,ESTABLISHED</div><div class="line">5      760 63840 ACCEPT     icmp --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">6        1    60 ACCEPT     all  --  lo     *       0.0.0.0/0            0.0.0.0/0</div><div class="line">7      977 58620 ACCEPT     tcp  --  *      *       0.0.0.0/0            0.0.0.0/0           state NEW tcp dpt:22</div><div class="line">8     3899 1194K REJECT     all  --  *      *       0.0.0.0/0            0.0.0.0/0           reject-with icmp-host-prohibited</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以看到，跟安全组相关的规则被重定向到neutron-openvswi-INPUT。 查看其规则，只有一条。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-INPUT</div><div class="line">Chain neutron-openvswi-INPUT (1 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1        0     0 neutron-openvswi-o583c7038<span class="_">-d</span>  all  --  *      *       0.0.0.0/0            0.0.0.0/0           PHYSDEV match --physdev-in tap583c7038-d3 --physdev-is-bridged</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;重定向到neutron-openvswi-o583c7038-d。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-o583c7038<span class="_">-d</span></div><div class="line">Chain neutron-openvswi-o583c7038<span class="_">-d</span> (2 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1     3894 1199K RETURN     udp  --  *      *       0.0.0.0/0            0.0.0.0/0           udp spt:68 dpt:67</div><div class="line">2     4282 1536K neutron-openvswi-s583c7038<span class="_">-d</span>  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">3        0     0 DROP       udp  --  *      *       0.0.0.0/0            0.0.0.0/0           udp spt:67 dpt:68</div><div class="line">4        0     0 DROP       all  --  *      *       0.0.0.0/0            0.0.0.0/0           state INVALID</div><div class="line">5     3971 1510K RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0           state RELATED,ESTABLISHED</div><div class="line">6      311 25752 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">7        0     0 neutron-openvswi-sg-fallback  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;如果是vm发出的dhcp请求，直接通过，否则转到neutron-openvswi-s583c7038-d。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-s583c7038<span class="_">-d</span></div><div class="line">Chain neutron-openvswi-s583c7038<span class="_">-d</span> (1 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1     4284 1537K RETURN     all  --  *      *       192.168.0.2          0.0.0.0/0           MAC FA:16:3E:9C:DC:3A</div><div class="line">2        0     0 DROP       all  --  *      *       0.0.0.0/0            0.0.0.0/0</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这条chain主要检查从vm发出来的网包，是否是openstack所分配的IP和MAC，如果不匹配，则禁止通过。这将防止利用vm上进行一些伪装地址的攻击。</p>
<h3 id="OUTPUT"><a href="#OUTPUT" class="headerlink" title="OUTPUT"></a>OUTPUT</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL OUTPUT</div><div class="line">Chain OUTPUT (policy ACCEPT 965K packets, 149M bytes)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1     481K  107M neutron-filter-top  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">2     481K  107M neutron-openvswi-OUTPUT  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;分别跳转到neutron-filter-top和neutron-openvswi-OUTPUT。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-filter-top</div><div class="line">Chain neutron-filter-top (2 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1     497K  112M neutron-openvswi-local  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;跳转到neutron-openvswi-local。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-OUTPUT</div><div class="line">Chain neutron-openvswi-OUTPUT (1 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;该chain目前无规则。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-local</div><div class="line">Chain neutron-openvswi-local (1 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;该chain目前也无规则。</p>
<h3 id="FORWARD"><a href="#FORWARD" class="headerlink" title="FORWARD"></a>FORWARD</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;FORWARD chain上主要实现安全组的功能。用户在配置缺省安全规则时候（例如允许ssh到vm，允许ping到vm），影响该chain。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL FORWARD</div><div class="line">Chain FORWARD (policy ACCEPT 0 packets, 0 bytes)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1    16203 5342K neutron-filter-top  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">2    16203 5342K neutron-openvswi-FORWARD  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">3        0     0 REJECT     all  --  *      *       0.0.0.0/0            0.0.0.0/0           reject-with icmp-host-prohibited</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;同样跳转到neutron-filter-top，无规则。跳转到neutron-openvswi-FORWARD。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-FORWARD</div><div class="line">Chain neutron-openvswi-FORWARD (1 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1     8170 2630K neutron-openvswi-sg-chain  all  --  *      *       0.0.0.0/0            0.0.0.0/0           PHYSDEV match --physdev-out tap583c7038-d3 --physdev-is-bridged</div><div class="line">2     8156 2729K neutron-openvswi-sg-chain  all  --  *      *       0.0.0.0/0            0.0.0.0/0           PHYSDEV match --physdev-in tap583c7038-d3 --physdev-is-bridged</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;neutron-openvswi-FORWARD将匹配所有进出tap-XXX端口的流量。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-sg-chain</div><div class="line">Chain neutron-openvswi-sg-chain (2 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1     8170 2630K neutron-openvswi-i583c7038<span class="_">-d</span>  all  --  *      *       0.0.0.0/0            0.0.0.0/0           PHYSDEV match --physdev-out tap583c7038-d3 --physdev-is-bridged</div><div class="line">2     8156 2729K neutron-openvswi-o583c7038<span class="_">-d</span>  all  --  *      *       0.0.0.0/0            0.0.0.0/0           PHYSDEV match --physdev-in tap583c7038-d3 --physdev-is-bridged</div><div class="line">3    12442 4163K ACCEPT     all  --  *      *       0.0.0.0/0            0.0.0.0/0</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;如果是网桥从tap-XXX端口发出到VM的流量，则跳转到neutron-openvswi-i9LETTERID；如果是从tap-XXX端口进入到网桥的（即vm发出来的）流量，则跳转到neutron-openvswi-o9LETTERID。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-i583c7038<span class="_">-d</span></div><div class="line">Chain neutron-openvswi-i583c7038<span class="_">-d</span> (1 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1        0     0 DROP       all  --  *      *       0.0.0.0/0            0.0.0.0/0           state INVALID</div><div class="line">2      400 43350 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0           state RELATED,ESTABLISHED</div><div class="line">3        1    60 RETURN     tcp  --  *      *       0.0.0.0/0            0.0.0.0/0           tcp dpt:22</div><div class="line">4        1    84 RETURN     icmp --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">5     3885 1391K RETURN     udp  --  *      *       192.168.0.3          0.0.0.0/0           udp spt:67 dpt:68</div><div class="line">6     3885 1197K neutron-openvswi-sg-fallback  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;neutron-openvswi-i9LETTERID允许安全组中配置的策略（允许ssh、ping等）和dhcp reply通过。默认的neutron-openvswi-sg-fallback将drop所有流量。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div></pre></td><td class="code"><pre><div class="line">iptables --line-numbers -vnL neutron-openvswi-o583c7038<span class="_">-d</span></div><div class="line">Chain neutron-openvswi-o583c7038<span class="_">-d</span> (2 references)</div><div class="line">num   pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">1     3886 1197K RETURN     udp  --  *      *       0.0.0.0/0            0.0.0.0/0           udp spt:68 dpt:67</div><div class="line">2     4274 1533K neutron-openvswi-s583c7038<span class="_">-d</span>  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">3        0     0 DROP       udp  --  *      *       0.0.0.0/0            0.0.0.0/0           udp spt:67 dpt:68</div><div class="line">4        0     0 DROP       all  --  *      *       0.0.0.0/0            0.0.0.0/0           state INVALID</div><div class="line">5     3963 1507K RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0           state RELATED,ESTABLISHED</div><div class="line">6      311 25752 RETURN     all  --  *      *       0.0.0.0/0            0.0.0.0/0</div><div class="line">7        0     0 neutron-openvswi-sg-fallback  all  --  *      *       0.0.0.0/0            0.0.0.0/0</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;neutron-openvswi-o9LETTERID将跳转到neutron-openvswi-s583c7038-d，允许DHCP Request和匹配VM的源IP和源MAC的流量通过。</p>
<h3 id="整体逻辑"><a href="#整体逻辑" class="headerlink" title="整体逻辑"></a>整体逻辑</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;整体逻辑如下图所示。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/14.png?raw=true" alt="14"></p>
<h2 id="快速查找安全组规则"><a href="#快速查找安全组规则" class="headerlink" title="快速查找安全组规则"></a>快速查找安全组规则</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;从前面分析可以看出，某个vm的安全组相关规则的chain的名字，跟vm的id的前9个字符有关。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;因此，要快速查找qbr-XXX上相关的iptables规则，可以用iptables -S列出（默认是filter表）所有链上的规则，其中含有id的链即为虚拟机相关的安全组规则。其中–physdev-in表示即将进入某个网桥的端口，–physdev-out表示即将从某个网桥端口发出。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">iptables -S |grep tap583c7038-d3</div><div class="line">-A neutron-openvswi-FORWARD -m physdev --physdev-out tap583c7038-d3 --physdev-is-bridged -j neutron-openvswi-sg-chain</div><div class="line">-A neutron-openvswi-FORWARD -m physdev --physdev-in tap583c7038-d3 --physdev-is-bridged -j neutron-openvswi-sg-chain</div><div class="line">-A neutron-openvswi-INPUT -m physdev --physdev-in tap583c7038-d3 --physdev-is-bridged -j neutron-openvswi-o583c7038<span class="_">-d</span></div><div class="line">-A neutron-openvswi-sg-chain -m physdev --physdev-out tap583c7038-d3 --physdev-is-bridged -j neutron-openvswi-i583c7038<span class="_">-d</span></div><div class="line">-A neutron-openvswi-sg-chain -m physdev --physdev-in tap583c7038-d3 --physdev-is-bridged -j neutron-openvswi-o583c7038<span class="_">-d</span></div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以看出，进出tap-XXX口的FORWARD链上的流量都被扔到了neutron-openvswi-sg-chain这个链，neutron-openvswi-sg-chain上是security group具体的实现（两条规则，访问虚拟机的流量扔给neutron-openvswi-i583c7038-d；从虚拟机出来的扔给neutron-openvswi-o583c7038-d）。</p>
<h3 id="其它"><a href="#其它" class="headerlink" title="其它"></a>其它</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;安全组在Havana版本中，默认是开启的，如果安装完毕后发现找不到qbr-*网桥，则可以检查在nova.conf里面是否设置以下内容：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">libvirt_vif_driver=nova.virt.libvirt.vif.LibvirtHybridOVSBridgeDriver</div></pre></td></tr></table></figure>
<h2 id="负载均衡即服务"><a href="#负载均衡即服务" class="headerlink" title="负载均衡即服务"></a>负载均衡即服务</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;负载均衡即服务（Load Balance as a Service，LBaaS）是一项网络高级服务。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;顾名思义，它允许租户动态的在自己的网络创建一个负载均衡设备。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;负载均衡，可以说是分布式系统中比较基础的组件，它接收前端过来的请求，然后将请求按照某种均衡的策略转发给后端资源池中的某个处理单元，以完成处理。进而可以实现高可用性和横向的扩展性。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;OpenStack Neutron 通过高级服务扩展的形式支持 LBaaS，目前默认是通过 HAProxy 软件来实现的。</p>
<h3 id="典型场景"><a href="#典型场景" class="headerlink" title="典型场景"></a>典型场景</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;典型的场景，租户创建了一个网络，并在其上分配了一个子网 10.0.0.0/24，租户试图通过一个负载均衡设备来实现多个虚拟机呈现统一业务。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;首先，启动两个虚机，分别分配到 IP 地址：10.0.0.2 和 10.0.0.4。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;OpenStack 的 LBaaS 实现中有三个重要概念：</p>
<ul>
<li>Pool</li>
<li>Member</li>
<li>Monitor</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，Pool 是要进行负载均衡的资源池（一般需要对应到某一个子网），可以指定负载均衡的提供机制（例如默认的 HAProxy）、均衡的协议（TCP、HTTP、HTTPS）和端口等、均衡的策略。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/15.png?raw=true" alt="15"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;定义完成 Pool 后可以往里面添加 Member（即虚拟机），并为各个成员分配权值，指定的目标端口等。 </p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/16.png?raw=true" alt="16"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;最后，定义一个 Monitor，负责定期探测成员的状态。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/17.png?raw=true" alt="17"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;例如，我们创建一个 Pool，添加子网 10.0.0.0/24，并添加启动的虚机 10.0.0.2 和 10.0.0.4 作为成员，创建一个 VIP（例如 10.0.0.250）来均衡 TCP 协议，端口 22 的请求。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这样，对地址 10.0.0.250 的 TCP 访问，只要是端口 22 的，会被自动负载均衡到虚机 10.0.0.2 和 10.0.0.4 上去。</p>
<h3 id="实现细节"><a href="#实现细节" class="headerlink" title="实现细节"></a>实现细节</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;跟大部分的高级服务一样，LBaaS 在网络节点上实现。</p>
<h3 id="qlbaas-命名空间"><a href="#qlbaas-命名空间" class="headerlink" title="qlbaas 命名空间"></a>qlbaas 命名空间</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在启动了 LBaaS 之后，网络节点上会多一个 qlbaas-xxx 命名空间，其中一个 tap 类型端口，绑定了我们定义的 VIP。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div></pre></td><td class="code"><pre><div class="line">sudo ip netns <span class="built_in">exec</span> qlbaas-574a31da-a28a-449f-8c9d-3d3687c3c02a ip addr</div><div class="line">1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default</div><div class="line">    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00</div><div class="line">    inet 127.0.0.1/8 scope host lo</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">    inet6 ::1/128 scope host</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">52: tapfb358342-59: &lt;BROADCAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UNKNOWN group default</div><div class="line">    link/ether fa:16:3e:68:77:37 brd ff:ff:ff:ff:ff:ff</div><div class="line">    inet 10.0.0.250/24 brd 10.0.0.255 scope global tapfb358342-59</div><div class="line">       valid_lft forever preferred_lft forever</div><div class="line">    inet6 fe80::f816:3eff:fe68:7737/64 scope link</div><div class="line">       valid_lft forever preferred_lft forever</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;该 tap 类型端口，会连接到网络节点的 br-int 网桥上。</p>
<h3 id="HAProxy"><a href="#HAProxy" class="headerlink" title="HAProxy"></a>HAProxy</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;此时在网络节点上查看进程，会发现一个 haproxy 进程。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">ps aux|grep haproxy</div><div class="line">nobody   17331  0.0  0.0  20376  1104 ?        Ss   15:08   0:00 haproxy -f /opt/stack/data/neutron/lbaas/574a31da-a28a-449f-8c9d-3d3687c3c02a/conf -p /opt/stack/data/neutron/lbaas/574a31da-a28a-449f-8c9d-3d3687c3c02a/pid -sf 17323</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;查看 HAProxy 的配置文件</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div></pre></td><td class="code"><pre><div class="line">cat /opt/stack/data/neutron/lbaas/574a31da-a28a-449f-8c9d-3d3687c3c02a/conf</div><div class="line">global</div><div class="line">        daemon</div><div class="line">        user nobody</div><div class="line">        group nogroup</div><div class="line">        <span class="built_in">log</span> /dev/<span class="built_in">log</span> local0</div><div class="line">        <span class="built_in">log</span> /dev/<span class="built_in">log</span> local1 notice</div><div class="line">        stats socket /opt/stack/data/neutron/lbaas/574a31da-a28a-449f-8c9d-3d3687c3c02a/sock mode 0666 level user</div><div class="line">defaults</div><div class="line">        <span class="built_in">log</span> global</div><div class="line">        retries 3</div><div class="line">        option redispatch</div><div class="line">        timeout connect 5000</div><div class="line">        timeout client 50000</div><div class="line">        timeout server 50000</div><div class="line">frontend df25900f-53b1-4aab-8436-8411ec710445</div><div class="line">        option tcplog</div><div class="line">        <span class="built_in">bind</span> 10.0.0.250:22</div><div class="line">        mode tcp</div><div class="line">        default_backend 574a31da-a28a-449f-8c9d-3d3687c3c02a</div><div class="line">backend 574a31da-a28a-449f-8c9d-3d3687c3c02a</div><div class="line">        mode tcp</div><div class="line">        balance leastconn</div><div class="line">        server 45481748-3104-462c-ae4b-fae0f4b08c20 10.0.0.2:22 weight 1</div><div class="line">        server 227b69fe-8685-4df6-b2a5-2f964c74886c 10.0.0.4:22 weight 1</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可见，里面的配置信息，跟我们之前配置的完全一致。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;当有请求访问 10.0.0.250:22 的时候，最终会到达该 qlbaas 命名空间，请求被 HAProxy 收到，按照负载均衡策略，转发给合适的虚机。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以通过 ssh 进行测试。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo ip netns <span class="built_in">exec</span> qrouter-40fff075-d3a2-477b-942c-6b1adb42e35e ssh 10.0.0.250</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在后端的虚机抓包，SSH 请求是 10.0.0.250 -&gt; 10.0.0.2，说明实际上是 HAProxy 转发了请求。</p>
<h3 id="其它问题"><a href="#其它问题" class="headerlink" title="其它问题"></a>其它问题</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;我们知道，虚拟机可以通过分配 Floating IP 的方式被外部直接访问到它内部的服务，那么经过负载均衡后，是否也能为 VIP 分配一个 floating IP 呢？</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;答案是肯定的。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以先手动申请到一个 Floating IP，在绑定的时候选择 VIP 对应的端口即可。这样，我们就可以对外呈现我们的高可用、动态扩展的服务了。</p>
<h2 id="防火墙即服务"><a href="#防火墙即服务" class="headerlink" title="防火墙即服务"></a>防火墙即服务</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;熟悉防火墙的都知道，防火墙一般放在网关上，用来隔离子网之间的访问。因此，防火墙即服务（FireWall as a Service）也是在网络节点上（具体说来是在路由器命名空间中）来实现。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;目前，OpenStack 中实现防火墙还是基于 Linux 系统自带的 iptables，所以大家对于其性能和功能就不要抱太大的期望了。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;一个可能混淆的概念是安全组（Security Group），安全组的对象是虚拟网卡，由L2 Agent来实现，比如neutron_openvswitch_agent 和 neutron_linuxbridge_agent，会在计算节点上通过配置 iptables 规则来限制虚拟网卡的进出访问。防火墙可以在安全组之前隔离外部过来的恶意流量，但是对于同个子网内部不同虚拟网卡间的通讯不能过滤（除非它要跨子网）。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以同时部署防火墙和安全组实现双重防护。</p>
<h3 id="典型场景-1"><a href="#典型场景-1" class="headerlink" title="典型场景"></a>典型场景</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;典型的场景，租户创建了一个网络，并在其上分配了一个子网 10.0.0.0/24，默认情况下，其它子网将不允许访问该子网。租户试图通过防火墙规则来允许外部网络对内部子网虚拟机 22 端口的访问。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;OpenStack 的 FWaaS 实现中有三个重要概念：</p>
<ul>
<li>Firewall</li>
<li>Policy</li>
<li>Rule</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，Firewall 会绑定到某个 Policy（因此必须先创建 Policy 之后才能创建一个 Firewall）。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/18.png?raw=true" alt="18"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;Policy 中可以包括若干条 Rule。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/19.png?raw=true" alt="19"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;Rule 即访问控制的规则，包括源和目的子网、源和目的端口、协议、行动等。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;例如，我们创建一个 Rule，允许其它网络对内部子网中虚机端口 22 的 TCP 请求。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/20.png?raw=true" alt="20"></p>
<h3 id="实现细节-1"><a href="#实现细节-1" class="headerlink" title="实现细节"></a>实现细节</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;防火墙由 L3 Agent 通过修改 iptables 规则来实现，具体规则在网络节点的路由器命名空间中，作用到该租户所有路由器的 qr-xxx 接口上。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在网络节点上查看其中的 iptables 规则，会发现多了两个 iptables 链，分别处理进出两个方向的流量，被 neutron-vpn-agen-FORWARD 引用。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div></pre></td><td class="code"><pre><div class="line">sudo ip netns <span class="built_in">exec</span> qrouter-40fff075-d3a2-477b-942c-6b1adb42e35e iptables -nvL</div><div class="line">Chain neutron-vpn-agen-iv4d8ed9898 (1 references)</div><div class="line"> pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">    0     0 DROP       all  --  *      *       0.0.0.0/0            0.0.0.0/0            state INVALID</div><div class="line">    0     0 ACCEPT     all  --  *      *       0.0.0.0/0            0.0.0.0/0            state RELATED,ESTABLISHED</div><div class="line">    0     0 ACCEPT     tcp  --  *      *       0.0.0.0/0            10.0.0.0/24          tcp spts:1:65535 dpt:22</div><div class="line"></div><div class="line">Chain neutron-vpn-agen-ov4d8ed9898 (1 references)</div><div class="line"> pkts bytes target     prot opt <span class="keyword">in</span>     out     <span class="built_in">source</span>               destination</div><div class="line">    0     0 DROP       all  --  *      *       0.0.0.0/0            0.0.0.0/0            state INVALID</div><div class="line">    0     0 ACCEPT     all  --  *      *       0.0.0.0/0            0.0.0.0/0            state RELATED,ESTABLISHED</div><div class="line">    0     0 ACCEPT     tcp  --  *      *       0.0.0.0/0            10.0.0.0/24          tcp spts:1:65535 dpt:22</div></pre></td></tr></table></figure>
<h3 id="其它问题-1"><a href="#其它问题-1" class="headerlink" title="其它问题"></a>其它问题</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其实像类似的高级服务，应该使用更为专业的实现，包括各种现成的防火墙硬件、软件实现。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;但是很可惜的是，Neutron 中防火墙的位置跟路由器结合的过于紧密，造成要替换就要全部替换掉，即使用外部的网关实现。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其它高级服务也存在类似的问题，特别是支持了服务链之后，耦合的更严重了。</p>
<h2 id="分布式路由"><a href="#分布式路由" class="headerlink" title="分布式路由"></a>分布式路由</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;OpenStack 用户可能会发现，按照 Neutron 原先的设计，所有网络服务都在网络节点上进行，这意味着大量的流量和处理，给网络节点带来了很大的压力。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这些处理的核心是路由器服务。任何需要跨子网的访问都需要路由器进行路由。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;很自然，能否让计算节点上也运行路由器服务？这个设计思路无疑是更为合理的，但具体实施起来需要诸多细节上的技术考量。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;为了降低网络节点的负载，同时提高可扩展性，OpenStack 自 Juno 版本开始正式引入了分布式路由（Distributed Virtual Router，DVR）特性（用户可以选择使用与否），来让计算节点自己来处理原先的大量东西向流量和非 SNAT 南北流量（有 floating IP 的 vm 跟外面的通信）。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这样网络节点只需要处理占到一部分的 SNAT （无 floating IP 的 vm 跟外面的通信）流量，大大降低了负载和整个系统对网络节点的依赖。很自然的，FWaaS 也可以跟着放到计算节点上。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;DHCP 服务、VPN 服务目前仍然需要集中在网络节点上进行。</p>
<h3 id="典型场景-2"><a href="#典型场景-2" class="headerlink" title="典型场景"></a>典型场景</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;从网络的访问看，涉及到路由服务的至少是需要跨子网的访问，又包括是否是同一机器、是否是涉及到外网（东西向 vs 南北向）。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;考虑下面几个跨子网路由的典型场景。</p>
<table>
<thead>
<tr>
<th>方向</th>
<th>同一机器</th>
<th>不同机器</th>
</tr>
</thead>
<tbody>
<tr>
<td>东西</td>
<td>本地网桥处理</td>
<td>本地东西路由器</td>
</tr>
<tr>
<td>南北</td>
<td>本地南北路由器 floating 转发</td>
<td>网络节点 SNAT 转发</td>
</tr>
</tbody>
</table>
<h4 id="东西向"><a href="#东西向" class="headerlink" title="东西向"></a>东西向</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;东西向意味着租户同一个数据中心内不同子网之间的互相访问。</p>
<h5 id="同一机器"><a href="#同一机器" class="headerlink" title="同一机器"></a>同一机器</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对于同一主机上的不同子网之间访问，路由器直接在 br-int 上转发即可，不需要经过外部网桥。</p>
<h5 id="不同机器"><a href="#不同机器" class="headerlink" title="不同机器"></a>不同机器</h5><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/21.png?raw=true" alt="21"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;如图所示，租户 T1 的两台虚拟机 VM1（计算节点 1）和 VM4（计算节点 2）分别属于不同子网，位于不同的计算节点。VM1 要访问 VM4，由计算节点 1 上的 IR1 起到路由器功能。返程的网包，则由计算节点 2 上的路由器 IR2 起作用。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;两个路由器的 id、内部接口、功能等其实都是一样的。即同一个路由器，但是实际上在多个计算节点上同时存在。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这里可能有人会想到，多台同样的路由器，如果都暴露在外部网络上，会出现冲突。例如当 VM1 的请求包离开计算节点 1 时，带的源 mac 是路由器目标接口的 mac，而这个 mac 在计算节点 2 上的路由器上同样存在。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;因此，需要拦截路由器对外的暴露信息。一个是让每个路由器只应答本机的 mac 请求；另一个是绝对不让带着路由器 mac 地址的包直接扔出去。实现上在 br-int 上进行拦截，修改其源 mac 为 tunnel 端口的 mac。同样的，计算节点 2 在 br-int 上拦截源 mac 为这个 tunnel 端口的 mac，替换为正常的子网网关的 mac，直接扔给目标虚拟机所在的主机。</p>
<h4 id="南北向"><a href="#南北向" class="headerlink" title="南北向"></a>南北向</h4><h5 id="无-floating-IP"><a href="#无-floating-IP" class="headerlink" title="无 floating IP"></a>无 floating IP</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这种情况（即 SNAT）下，跟传统模式很类似，首先来看外部访问内部子网。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/22.png?raw=true" alt="22"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;租户 T2 在外部，通过默认的 SNAT 网关访问内部子网的 vm VM1。此时，网络节点上的 T2-SNAT 起到路由器的作用</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/23.png?raw=true" alt="23"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;反过来，租户 T2 内部子网的 vm VM1 试图访问外部网络，则仍然经过网络节点上的 T2-SNAT 路由器。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;为何这种情况下必须从网络节点走？这是因为对于外部网络来说，看到的都是外部接口的地址，这个地址只有一个。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;当然，如果以后每个计算节点上都可以带有这样一个 SNAT 默认外部地址的话，这种情况下的流量也是可以直接从计算节点出去的。</p>
<h5 id="有-floating-IP"><a href="#有-floating-IP" class="headerlink" title="有 floating IP"></a>有 floating IP</h5><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/24.png?raw=true" alt="24"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这种情况下，计算节点上的专门负责的外部路由器将负责进行转发，即计算节点 1 上的 IR2 和计算节点 2 上的 IR1。</p>
<h3 id="网络节点-3"><a href="#网络节点-3" class="headerlink" title="网络节点"></a>网络节点</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;服务基本没变动，除了 L3 服务需要配置为 dvr_snat 模式。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;命名空间上会多一个专门的 snat-xxx 命名空间，处理来自计算节点的无 floating IP 的南北向流量。</p>
<h3 id="计算节点-3"><a href="#计算节点-3" class="headerlink" title="计算节点"></a>计算节点</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;需要额外启用 l3_agent（dvr 模式），以及 metadata agent。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其实，跟传统情况下的网络节点十分类似。每个东西向路由器有自己的命名空间，负责跨子网的转发。另外，多一个 floating 路由器，专门负责经由 floating 地址的南北向转发。</p>
<h4 id="东西流量"><a href="#东西流量" class="headerlink" title="东西流量"></a>东西流量</h4><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/25.png?raw=true" alt="25"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;如上图所示，租户两个子网，红色和绿色，分别有 vm1 和 vm2，位于节点 cn1 和 cn2 上。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;vm1 访问 vm2 的网包如步骤 1-6，整个过程 ip 保持不变。</p>
<ul>
<li>原始包，vm1 访问 vm2，目的 mac 为本地（红网）的路由器网关接口 r1 red mac；</li>
<li>经过 br-int-cn1 转发，该网包通过本地（红网）网关接口扔给本地路由器 r1。</li>
<li>r1 根据路由规则，经过到绿网的接口发出，此时网包的源 mac 改为绿网的网关接口 r1 grn mac，目的 mac 改为 vm2 mac，并且带上绿网的本地 vlan tag；</li>
<li>网包发给 br-tun-cn1 进行 tunnel，扔出去之前，将源 mac 替换为跟节点相关的特定 mac dvr cn1 mac，之后带着目标子网（绿网）的 外部 tunnel id 扔出去（实现可以为 vlan、vxlan、gre 等，功能都是一样的）；</li>
<li>节点 cn2 的网桥 br-tun-cn2 会从 tunnel 收到这个包，解封包，带上本地 vlan tag，最终抵达网桥 br-int-cn2；</li>
<li>br-int-cn2 上替换网包的源 mac（此时为 dvr-cn1-mac）为本地路由器的绿网网关接口，然后发给 vm2。</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;返回包的过程正好是反过来。虽然实现上略复杂，但整个过程还是比较清晰的，保证 vm1 和 vm2 感觉到的都是直接跟路由器的接口相连（分别为红网网关接口和绿网网关接口）。</p>
<h4 id="南北流量"><a href="#南北流量" class="headerlink" title="南北流量"></a>南北流量</h4><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/26.png?raw=true" alt="26"></p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;所不同的是，单独有一个 qfloat-XXX 路由器（也在一个独立命名空间中）来负责处理带有 floating IP 的南北向流量。</p>
<p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/27.png?raw=true" alt="27"></p>
<h3 id="流表规则"><a href="#流表规则" class="headerlink" title="流表规则"></a>流表规则</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;为了实现上面描述的这些功能，需要下面几种流表规则。</p>
<h4 id="所有对网关的-arp-request-不能出到外部网络"><a href="#所有对网关的-arp-request-不能出到外部网络" class="headerlink" title="所有对网关的 arp request 不能出到外部网络"></a>所有对网关的 arp request 不能出到外部网络</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 tunnel 网桥上看到的一律丢弃。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">DVR PROCESS Table 1 (New table <span class="keyword">for</span> dvr):</div><div class="line">table=1, priority=4, dl_vlan= red1-L-vlan, dl_type=arp, ar_tpa= r1-red-ip actions: drop</div><div class="line">table=1, priority=4, dl_vlan= grn1-L-vlan, dl_type=arp, ar_tpa= r1-grn-ip actions: drop</div></pre></td></tr></table></figure>
<h4 id="所有发往网关的包也不能出到外部网络"><a href="#所有发往网关的包也不能出到外部网络" class="headerlink" title="所有发往网关的包也不能出到外部网络"></a>所有发往网关的包也不能出到外部网络</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 tunnel 网桥上看到的一律丢弃。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">DVR PROCESS Table 1 (New table <span class="keyword">for</span> dvr):</div><div class="line">table=1, priority=2, dl_vlan=red2_L_vlan, dl_dst=r1-red-mac, actions: drop</div><div class="line">table=1, priority=2 , dl_vlan=grn2_L_vlan, dl_dst=r1-grn-mac, actions: drop</div></pre></td></tr></table></figure>
<h4 id="所有从路由器接口路由出去的包，需要修改源-mac-为特殊-mac"><a href="#所有从路由器接口路由出去的包，需要修改源-mac-为特殊-mac" class="headerlink" title="所有从路由器接口路由出去的包，需要修改源 mac 为特殊 mac"></a>所有从路由器接口路由出去的包，需要修改源 mac 为特殊 mac</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 tunnel 网桥上进行处理。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">DVR PROCESS Table 1 (New table <span class="keyword">for</span> dvr):</div><div class="line">table=1, priority=1, dl_vlan=red2_L_vlan, dl_src=r1-red-mac, actions: mod_dl_src=dvr-cn1-mac, resubmit(,2)</div><div class="line">table=1, priority=1, dl_vlan=grn2_L_vlan, dl_src=r1-grn-mac, actions: mod_dl_src=dvr-cn1-mac, resubmit (,2)</div></pre></td></tr></table></figure>
<h4 id="同样，收到外部特殊源-mac-的包要修改为正确的路由器接口-mac"><a href="#同样，收到外部特殊源-mac-的包要修改为正确的路由器接口-mac" class="headerlink" title="同样，收到外部特殊源 mac 的包要修改为正确的路由器接口 mac"></a>同样，收到外部特殊源 mac 的包要修改为正确的路由器接口 mac</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 integration 网桥上进行处理。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div></pre></td><td class="code"><pre><div class="line">Table 0: (Local switching table)</div><div class="line">table=0, priority=2, in_port=patch-tun, dl_src=dvr-cn1-mac actions: goto table 1</div><div class="line">table=0, priority=1, actions: output-&gt;NORMAL</div><div class="line"></div><div class="line">Table 1: (DVR_TO_LOCALMAC table)</div><div class="line">table=1, priority=2, dl_vlan=grn2-L-vlan, nw_dst=grn-subnet actions: strip_vlan, mod_dl_src=r1-grn-MAC,output-&gt;port-vm2</div><div class="line">table=1, priority=1 actions: drop</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;此外，还有一些优化措施，包括：<br>采用 L2 pre-population 技术，提前把相关计算节点的地址关系放到本地的 FDB 表中，减少外部广播。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 integration 网桥上，采用组表来调整规则顺序等。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">Table 1: (DVR_TO_LOCALMAC table)</div><div class="line">table=1, priority=2, dl_vlan=grn2-L-vlan, nw_dst=grn-subnet actions: strip_vlan, mod_dl_src=r1-grn-MAC,output-&gt;port-vm2</div></pre></td></tr></table></figure>
<h3 id="注意事项"><a href="#注意事项" class="headerlink" title="注意事项"></a>注意事项</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这里面比较重要的地方，是允许有多个同样的网关存在于多个计算节点（同一子网跨多个物理节点导致）的情况下，保证：</p>
<ul>
<li>要让本地的请求找到本地的路由器；</li>
<li>要避免路由器的接口 mac 地址直接暴露到外部网络上。</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;要解决这两个问题，首先是合理处理好本地的 ARP 请求，让本地对网关的请求拦截发给本地的路由器。同时，路由器路由后的网包，发到外部网络之前，先用一个跟节点绑定的特殊 mac 替换掉源 mac。这个 mac 地址是控制器为每个计算节点单独分配的唯一地址。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;如果不允许同一子网跨多个物理节点，则每个节点上的路由器就不会冲突，设计起来就简单的多，比如 <a href="http://blog.csdn.net/yeasy/article/details/46443933" target="_blank" rel="external">kubernetes 中的网络</a>。</p>
<h3 id="配置"><a href="#配置" class="headerlink" title="配置"></a>配置</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;要启用 DVR，比较简单，分别在各个节点的网络配置文件上做如下修改或添加。</p>
<h4 id="Neutron-Server"><a href="#Neutron-Server" class="headerlink" title="Neutron Server"></a>Neutron Server</h4><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">/etc/neutron/neutron.conf</div><div class="line">router_distributed = True</div></pre></td></tr></table></figure>
<h4 id="L3-Agent"><a href="#L3-Agent" class="headerlink" title="L3 Agent"></a>L3 Agent</h4><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">/etc/neutron/l3_agent.ini</div><div class="line">agent_mode = [dvr_snat | dvr | legacy ]</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;网络节点上配置为 dvr_snat，计算节点上配置为 dvr。</p>
<h4 id="L2-Agent"><a href="#L2-Agent" class="headerlink" title="L2 Agent"></a>L2 Agent</h4><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line">/etc/neutron/plugins/ml2/ml2_conf.ini</div><div class="line">[ml2]</div><div class="line">mechanism_drivers = openvswitch,linuxbridge,l2population</div><div class="line"></div><div class="line">[agent]</div><div class="line">l2_population = True</div><div class="line">tunnel_types = vxlan</div><div class="line">enable_distributed_routing = True</div></pre></td></tr></table></figure>
<h3 id="实现细节-2"><a href="#实现细节-2" class="headerlink" title="实现细节"></a>实现细节</h3><h4 id="添加路由器接口"><a href="#添加路由器接口" class="headerlink" title="添加路由器接口"></a>添加路由器接口</h4><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/28.png?raw=true" alt="28"></p>
<h4 id="删除路由器接口"><a href="#删除路由器接口" class="headerlink" title="删除路由器接口"></a>删除路由器接口</h4><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/29.png?raw=true" alt="29"></p>
<h4 id="启动新的虚拟机节点"><a href="#启动新的虚拟机节点" class="headerlink" title="启动新的虚拟机节点"></a>启动新的虚拟机节点</h4><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/30.png?raw=true" alt="30"></p>
<h4 id="删除虚拟机节点"><a href="#删除虚拟机节点" class="headerlink" title="删除虚拟机节点"></a>删除虚拟机节点</h4><p><img src="https://github.com/hcldirgit/image/blob/master/%E6%B7%B1%E5%85%A5%E7%90%86%E8%A7%A3%20Neutron%20--%20OpenStack%20%E7%BD%91%E7%BB%9C%E5%AE%9E%E7%8E%B0/31.png?raw=true" alt="31"></p>
<h3 id="实现细节-3"><a href="#实现细节-3" class="headerlink" title="实现细节"></a>实现细节</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;以 OpenvSwitch plugin 为例，主要在 neutron\plugins\openvswitch\agent\ovs_dvr_neutron_agent.py 文件中。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;OVSDVRNeutronAgent 类是本地处理的 agent，启动后会在三个网桥上添加初始化的规则，主要过程如下。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">def setup_dvr_flows(self):</div><div class="line">    self.setup_dvr_flows_on_integ_br()</div><div class="line">    self.setup_dvr_flows_on_tun_br()</div><div class="line">    self.setup_dvr_flows_on_phys_br()</div><div class="line">    self.setup_dvr_mac_flows_on_all_brs()</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，br-int 是本地交换网桥；br-tun 是跟其它计算节点通信的承载网桥；br-phy 是跟外部公共网络通信的网桥。</p>
<h4 id="integration-网桥规则添加"><a href="#integration-网桥规则添加" class="headerlink" title="integration 网桥规则添加"></a>integration 网桥规则添加</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;integrate 网桥负责本地同一子网内的网包交换。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要过程为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div></pre></td><td class="code"><pre><div class="line">Add a canary flow to int_br to track OVS restarts</div><div class="line">        self.int_br.add_flow(table=constants.CANARY_TABLE, priority=0,</div><div class="line">                             actions=<span class="string">"drop"</span>)</div><div class="line"></div><div class="line">        <span class="comment"># Insert 'drop' action as the default for Table DVR_TO_SRC_MAC</span></div><div class="line">        self.int_br.add_flow(table=constants.DVR_TO_SRC_MAC,</div><div class="line">                             priority=1,</div><div class="line">                             actions=<span class="string">"drop"</span>)</div><div class="line"></div><div class="line">        self.int_br.add_flow(table=constants.DVR_TO_SRC_MAC_VLAN,</div><div class="line">                             priority=1,</div><div class="line">                             actions=<span class="string">"drop"</span>)</div><div class="line"></div><div class="line">        <span class="comment"># Insert 'normal' action as the default for Table LOCAL_SWITCHING</span></div><div class="line">        self.int_br.add_flow(table=constants.LOCAL_SWITCHING,</div><div class="line">                             priority=1,</div><div class="line">                             actions=<span class="string">"normal"</span>)</div><div class="line"></div><div class="line">        <span class="keyword">for</span> physical_network <span class="keyword">in</span> self.bridge_mappings:</div><div class="line">            self.int_br.add_flow(table=constants.LOCAL_SWITCHING,</div><div class="line">                                 priority=2,</div><div class="line">                                 in_port=self.int_ofports[physical_network],</div><div class="line">                                 actions=<span class="string">"drop"</span>)</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;首先，网桥上采用了多个流表，定义在 openvswitch/common 目录下的 constants.py。主要流表包括：</p>
<ul>
<li>LOCAL_SWITCHING 表：0，顾名思义，用于本地的交换。</li>
<li>CANARY 表：23</li>
<li>DVR_TO_SRC_MAC 表：1</li>
<li>DVR_TO_SRC_MAC_VLAN 表：2</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;上面的代码配置 LOCAL_SWITCHING 表默认规则为 NORMAL（作为本地的正常学习交换机），并不允许宿主机本地端口进来的包。配置其它表默认行为为丢弃。</p>
<h4 id="tunnel-网桥规则添加"><a href="#tunnel-网桥规则添加" class="headerlink" title="tunnel 网桥规则添加"></a>tunnel 网桥规则添加</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要过程为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div></pre></td><td class="code"><pre><div class="line">self.tun_br.add_flow(priority=1,</div><div class="line">                     in_port=self.patch_int_ofport,</div><div class="line">                     actions=<span class="string">"resubmit(,%s)"</span> %</div><div class="line">                     constants.DVR_PROCESS)</div><div class="line"></div><div class="line"><span class="comment"># table-miss should be sent to learning table</span></div><div class="line">self.tun_br.add_flow(table=constants.DVR_NOT_LEARN,</div><div class="line">                     priority=0,</div><div class="line">                     actions=<span class="string">"resubmit(,%s)"</span> %</div><div class="line">                     constants.LEARN_FROM_TUN)</div><div class="line"></div><div class="line">self.tun_br.add_flow(table=constants.DVR_PROCESS,</div><div class="line">                     priority=0,</div><div class="line">                     actions=<span class="string">"resubmit(,%s)"</span> %</div><div class="line">                     constants.PATCH_LV_TO_TUN)</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要流表包括：</p>
<ul>
<li>DVR_PROCESS 表：1，顾名思义，用于 DVR 处理。</li>
<li>DVR_NOT_LEARN 表：9。</li>
<li>LEARN_FROM_TUN 表：10，学习从 tunnel 收到的网包。</li>
<li>PATCH_LV_TO_TUN 表：2。</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;所有从 br-int 过来的网包，发到 DVR_PROCESS 表处理。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;DVR_NOT_LEARN 表，默认将网包发给表 LEARN_FROM_TUN。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;DVR_PROCESS 表，默认将网包发给表 PATCH_LV_TO_TUN。</p>
<h4 id="physical-网桥规则添加"><a href="#physical-网桥规则添加" class="headerlink" title="physical 网桥规则添加"></a>physical 网桥规则添加</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要过程为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">for</span> physical_network <span class="keyword">in</span> self.bridge_mappings:</div><div class="line">            self.phys_brs[physical_network].add_flow(priority=2,</div><div class="line">                in_port=self.phys_ofports[physical_network],</div><div class="line">                actions=<span class="string">"resubmit(,%s)"</span> %</div><div class="line">                constants.DVR_PROCESS_VLAN)</div><div class="line">            self.phys_brs[physical_network].add_flow(priority=1,</div><div class="line">                actions=<span class="string">"resubmit(,%s)"</span> %</div><div class="line">                constants.DVR_NOT_LEARN_VLAN)</div><div class="line">            self.phys_brs[physical_network].add_flow(</div><div class="line">                table=constants.DVR_PROCESS_VLAN,</div><div class="line">                priority=0,</div><div class="line">                actions=<span class="string">"resubmit(,%s)"</span> %</div><div class="line">                constants.LOCAL_VLAN_TRANSLATION)</div><div class="line">            self.phys_brs[physical_network].add_flow(</div><div class="line">                table=constants.LOCAL_VLAN_TRANSLATION,</div><div class="line">                priority=2,</div><div class="line">                in_port=self.phys_ofports[physical_network],</div><div class="line">                actions=<span class="string">"drop"</span>)</div><div class="line">            self.phys_brs[physical_network].add_flow(</div><div class="line">                table=constants.DVR_NOT_LEARN_VLAN,</div><div class="line">                priority=1,</div><div class="line">                actions=<span class="string">"NORMAL"</span>)</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;对所有的连接到外部的物理网桥都添加类似规则。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要流表包括：</p>
<ul>
<li>DVR_PROCESS_VLAN：1</li>
<li>LOCAL_VLAN_TRANSLATION：2</li>
<li>DVR_NOT_LEARN_VLAN：3。</li>
</ul>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;默认从 br-int 来的所有网包发送给表 DVR_PROCESS_VLAN。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;DVR_PROCESS_VLAN 表，默认发给表 LOCAL_VLAN_TRANSLATION。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;LOCAL_VLAN_TRANSLATION 表丢弃从 br-int 来的所有网包。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其他所有网包发送给表 DVR_NOT_LEARN_VLAN。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;表 DVR_NOT_LEARN_VLAN 默认规则为 NORMAL。</p>
<h4 id="所有网桥上添加-mac-规则"><a href="#所有网桥上添加-mac-规则" class="headerlink" title="所有网桥上添加 mac 规则"></a>所有网桥上添加 mac 规则</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;主要过程为</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div></pre></td><td class="code"><pre><div class="line"><span class="keyword">for</span> physical_network <span class="keyword">in</span> self.bridge_mappings:</div><div class="line">    self.int_br.add_flow(table=constants.LOCAL_SWITCHING,</div><div class="line">        priority=4,</div><div class="line">        in_port=self.int_ofports[physical_network],</div><div class="line">        dl_src=mac[<span class="string">'mac_address'</span>],</div><div class="line">        actions=<span class="string">"resubmit(,%s)"</span> %</div><div class="line">        constants.DVR_TO_SRC_MAC_VLAN)</div><div class="line">    self.phys_brs[physical_network].add_flow(</div><div class="line">        table=constants.DVR_NOT_LEARN_VLAN,</div><div class="line">        priority=2,</div><div class="line">        dl_src=mac[<span class="string">'mac_address'</span>],</div><div class="line">        actions=<span class="string">"output:%s"</span> %</div><div class="line">        self.phys_ofports[physical_network])</div><div class="line"></div><div class="line"><span class="keyword">if</span> self.enable_tunneling:</div><div class="line">    <span class="comment"># Table 0 (default) will now sort DVR traffic from other</span></div><div class="line">    <span class="comment"># traffic depending on in_port</span></div><div class="line">    self.int_br.add_flow(table=constants.LOCAL_SWITCHING,</div><div class="line">                         priority=2,</div><div class="line">                         in_port=self.patch_tun_ofport,</div><div class="line">                         dl_src=mac[<span class="string">'mac_address'</span>],</div><div class="line">                         actions=<span class="string">"resubmit(,%s)"</span> %</div><div class="line">                         constants.DVR_TO_SRC_MAC)</div><div class="line">    <span class="comment"># Table DVR_NOT_LEARN ensures unique dvr macs in the cloud</span></div><div class="line">    <span class="comment"># are not learnt, as they may</span></div><div class="line">    <span class="comment"># result in flow explosions</span></div><div class="line">    self.tun_br.add_flow(table=constants.DVR_NOT_LEARN,</div><div class="line">                     priority=1,</div><div class="line">                     dl_src=mac[<span class="string">'mac_address'</span>],</div><div class="line">                     actions=<span class="string">"output:%s"</span> %</div><div class="line">                     self.patch_int_ofport)</div><div class="line">self.registered_dvr_macs.add(mac[<span class="string">'mac_address'</span>])</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;这里主要是添加对其他主机上的 DVR 网关发过来的网包的处理。</p>
<h5 id="外部网络为-vlan-的情况下"><a href="#外部网络为-vlan-的情况下" class="headerlink" title="外部网络为 vlan 的情况下"></a>外部网络为 vlan 的情况下</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-int 网桥上，表 LOCAL_SWITCHING 添加优先级为 4 的流，如果从外面过来的网包，MAC 源地址是其他的 DVR 网关地址，则扔给表 DVR_TO_SRC_MAC_VLAN 处理。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;br-eth 网桥上，表 DVR_NOT_LEARN_VLAN 则添加优先级为 2 的规则，MAC 源地址是其他的 DVR 网关地址，则发给 br-int。</p>
<h5 id="外部网络为-tunnel-的情况下"><a href="#外部网络为-tunnel-的情况下" class="headerlink" title="外部网络为 tunnel 的情况下"></a>外部网络为 tunnel 的情况下</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;类似，分别在 br-int 网桥的 LOCAL_SWITCHING 表和 br-tun 网桥的 DVR_NOT_LEARN 表添加流，优先级分别改为 2 和 1。</p>
<h2 id="工具"><a href="#工具" class="headerlink" title="工具"></a>工具</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;由于 OpenStack 自身的各个组件都是松耦合关系，常见的部署也都是分布式部署，造成要深入体会 Neutron 的工作过程，或者进行故障诊断，往往涉及到多个组件，是否繁琐。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;本章介绍一些工具，提高操作和开发的效率。</p>
<h3 id="easyOVS"><a href="#easyOVS" class="headerlink" title="easyOVS"></a>easyOVS</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;<a href="https://github.com/yeasy/easyOVS" target="_blank" rel="external">easyOVS</a> 是一个开源的 OpenvSwitch 虚拟交换机管理工具。使用它，用户可以很轻松的对 OpenvSwitch 的网桥、端口等进行查看，同时它深度整合了 OpenStack （支持 Havana 版本到 Juno 版本） 中网络相关的信息，也是一个十分强大的 Neutron 中各个组件的监测工具。</p>
<h4 id="主要功能一览"><a href="#主要功能一览" class="headerlink" title="主要功能一览"></a>主要功能一览</h4><ul>
<li>支持 OpenvSwitch 版本 1.4.6 ~ 2.0.2，OpenStack Havana 到 Juno 版本；</li>
<li>支持操作系统环境报 Ubuntu、Debian、CentOS、Fedora 和 Redhat；</li>
<li>输出结果经过处理，支持彩色输出，十分简洁易读；</li>
<li>开启 OpenStack 支持，可以获取端口的地址、mac、vlan 甚至虚拟机关联的 iptables 规则等信息；</li>
<li>对流表操作语法更加简洁，并支持通过 id 进行删除；</li>
<li>支持 tab 自动补全；</li>
<li>支持通过 -m ‘cmd’ 来直接运行命令，无需进入 CLI 操作。</li>
</ul>
<h4 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;安装十分简单，一行代码搞定。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">git <span class="built_in">clone</span> https://github.com/yeasy/easyOVS.git &amp;&amp; sudo bash ./easyOVS/util/install.sh</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;安装成功后，可以使用</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo easyovs</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;进入操作界面。</p>
<h4 id="打开-OpenStack-支持"><a href="#打开-OpenStack-支持" class="headerlink" title="打开 OpenStack 支持"></a>打开 OpenStack 支持</h4><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;由于 OpenStack 组件信息获取需要有相关的认证信息，因此需要在环境变量或者配置文件中进行指定。</p>
<h5 id="环境变量"><a href="#环境变量" class="headerlink" title="环境变量"></a>环境变量</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;可以在用户目录的 .bashrc 文件中加入</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line"><span class="built_in">export</span> OS_USERNAME=demo</div><div class="line"><span class="built_in">export</span> OS_TENANT_NAME=demo</div><div class="line"><span class="built_in">export</span> OS_PASSWORD=admin</div><div class="line"><span class="built_in">export</span> OS_AUTH_URL=http://127.0.0.1:5000/v2.0/</div></pre></td></tr></table></figure>
<h5 id="配置文件"><a href="#配置文件" class="headerlink" title="配置文件"></a>配置文件</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;默认的配置文件在 /etc/easyovs.conf，替换为相应的认证信息即可。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div></pre></td><td class="code"><pre><div class="line">[OS]</div><div class="line">auth_url = http://127.0.0.1:5000/v2.0</div><div class="line">username = demo</div><div class="line">password = admin</div><div class="line">tenant_name = demo</div></pre></td></tr></table></figure>
<h4 id="操作命令"><a href="#操作命令" class="headerlink" title="操作命令"></a>操作命令</h4><h5 id="help"><a href="#help" class="headerlink" title="help"></a>help</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;显示帮助信息.</p>
<h5 id="list"><a href="#list" class="headerlink" title="list"></a>list</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;列出本地的 OpenvSwitch 网桥，例如</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; list</div><div class="line">s1</div><div class="line"> Port:        s1-eth2 s1 s1-eth1</div><div class="line"> Interface:    s1-eth2 s1 s1-eth1</div><div class="line"> Controller:ptcp:6634 tcp:127.0.0.1:6633</div><div class="line"> Fail_Mode:    secure</div><div class="line">s2</div><div class="line"> Port:        s2 s2-eth3 s2-eth2 s2-eth1</div><div class="line"> Interface:    s2 s2-eth3 s2-eth2 s2-eth1</div><div class="line"> Controller:tcp:127.0.0.1:6633 ptcp:6635</div><div class="line"> Fail_Mode:    secure</div><div class="line">s3</div><div class="line"> Port:        s3-eth1 s3-eth3 s3-eth2 s3</div><div class="line"> Interface:    s3-eth1 s3-eth3 s3-eth2 s3</div><div class="line"> Controller:ptcp:6636 tcp:127.0.0.1:6633</div><div class="line"> Fail_Mode:    secure</div></pre></td></tr></table></figure>
<h5 id="show"><a href="#show" class="headerlink" title="show"></a>show</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; [bridge|default] show</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;显示某个网桥上的端口信息，例如</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; br-int show</div><div class="line">br-int</div><div class="line">Intf                Port        Vlan    Type        vmIP            vmMAC</div><div class="line">int-br-eth0         15</div><div class="line">qvo260209fa-72      11          1                   192.168.0.4     fa:16:3e:0f:17:04</div><div class="line">qvo583c7038-d3      2           1                   192.168.0.2     fa:16:3e:9c:dc:3a</div><div class="line">qvo8bf9cba2-3f      9           1                   192.168.0.5     fa:16:3e:a2:2f:0e</div><div class="line">qvod4de9fe0-6d      8           2                   10.0.0.2        fa:16:3e:38:2b:2e</div><div class="line">br-int              LOCAL               internal</div></pre></td></tr></table></figure>
<h5 id="dump"><a href="#dump" class="headerlink" title="dump"></a>dump</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; [bridge|default] dump</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;显示网桥上绑定的流表规则，例如</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; s1 dump</div><div class="line">ID TAB PKT       PRI   MATCH                                                       ACT</div><div class="line">0  0   0         2400  dl_dst=ff:ff:ff:ff:ff:ff                                    CONTROLLER:65535</div><div class="line">1  0   0         2400  arp                                                         CONTROLLER:65535</div><div class="line">2  0   0         2400  dl_type=0x88cc                                              CONTROLLER:65535</div><div class="line">3  0   0         2400  ip,nw_proto=2                                               CONTROLLER:65535</div><div class="line">4  0   0         801   ip                                                          CONTROLLER:65535</div><div class="line">5  0   2         800</div></pre></td></tr></table></figure>
<h5 id="addflow"><a href="#addflow" class="headerlink" title="addflow"></a>addflow</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; [bridge|default] addflow [match] actions=[action]</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;添加一条流到网桥，例如</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; br-int addflow priority=3 ip actions=OUTPUT:1</div></pre></td></tr></table></figure>
<h5 id="delflow"><a href="#delflow" class="headerlink" title="delflow"></a>delflow</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; [bridge|default] delflow id1 id2...</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;从网桥删除流，其中 id 信息可以从 dump 的结果中拿到.</p>
<h5 id="set"><a href="#set" class="headerlink" title="set"></a>set</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; bridge <span class="built_in">set</span></div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;指定默认网桥，同时进入网桥操作模式，指定后进行操作可以忽略网桥信息。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; <span class="built_in">set</span> br-int</div><div class="line">Set the default bridge to br-int.</div><div class="line">EasyOVS: br-int&gt;</div></pre></td></tr></table></figure>
<h5 id="exit"><a href="#exit" class="headerlink" title="exit"></a>exit</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; <span class="built_in">exit</span></div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;退出网桥模式，或者退出程序.</p>
<h5 id="get"><a href="#get" class="headerlink" title="get"></a>get</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; get</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在网桥模式下，获取当前的网桥名称.</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">EasyOVS: br-int&gt; get</div><div class="line">Current default bridge is br-int</div></pre></td></tr></table></figure>
<h5 id="ipt"><a href="#ipt" class="headerlink" title="ipt"></a>ipt</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; ipt vm_ip1, vm_ip2...</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;给定虚拟机 IP 地址，显示与它相关的 iptables 规则。需要启用 OpenStack 支持。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; ipt 192.168.0.2 192.168.0.4</div><div class="line"><span class="comment">## IP = 192.168.0.2, port = qvo583c7038-d ##</span></div><div class="line">    PKTS    SOURCE          DESTINATION     PROT  OTHER</div><div class="line"><span class="comment">#IN:</span></div><div class="line">     672    all             all             all   state RELATED,ESTABLISHED</div><div class="line">       0    all             all             tcp   tcp dpt:22</div><div class="line">       0    all             all             icmp</div><div class="line">       0    192.168.0.4     all             all</div><div class="line">       3    192.168.0.5     all             all</div><div class="line">       8    10.0.0.2        all             all</div><div class="line">   85784    192.168.0.3     all             udp   udp spt:67 dpt:68</div><div class="line"><span class="comment">#OUT:</span></div><div class="line">    196K    all             all             udp   udp spt:68 dpt:67</div><div class="line">   86155    all             all             all   state RELATED,ESTABLISHED</div><div class="line">    1241    all             all             all</div><div class="line"><span class="comment">#SRC_FILTER:</span></div><div class="line">   59163    192.168.0.2     all             all   MAC FA:16:3E:9C:DC:3A</div><div class="line"><span class="comment">## IP = 192.168.0.4, port = qvo260209fa-7 ##</span></div><div class="line">    PKTS    SOURCE          DESTINATION     PROT  OTHER</div><div class="line"><span class="comment">#IN:</span></div><div class="line">      73    all             all             all   state RELATED,ESTABLISHED</div><div class="line">       0    all             all             tcp   tcp dpt:22</div><div class="line">       0    all             all             icmp</div><div class="line">       0    192.168.0.2     all             all</div><div class="line">       0    192.168.0.5     all             all</div><div class="line">       0    10.0.0.2        all             all</div><div class="line">   11331    192.168.0.3     all             udp   udp spt:67 dpt:68</div><div class="line"><span class="comment">#OUT:</span></div><div class="line">   30034    all             all             udp   udp spt:68 dpt:67</div><div class="line">   11377    all             all             all   state RELATED,ESTABLISHED</div><div class="line">      12    all             all             all</div><div class="line"><span class="comment">#SRC_FILTER:</span></div><div class="line">    9859    192.168.0.4     all             all   MAC FA:16:3E:0F:17:04</div></pre></td></tr></table></figure>
<h5 id="query"><a href="#query" class="headerlink" title="query"></a>query</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; query port_ip, port_id...</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;给定某个的端口的 IP 地址，或者部分端口 id 信息，显示该端口相关的完整信息。需要启用 OpenStack 支持。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; query 10.0.0.2,c4493802</div><div class="line"><span class="comment">## port_id = f47c62b0-dbd2-4faa-9cdd-8abc886ce08f</span></div><div class="line">status: ACTIVE</div><div class="line">name:</div><div class="line">allowed_address_pairs: []</div><div class="line">admin_state_up: True</div><div class="line">network_id: ea3928dc-b1fd-4a1a-940e-82b8c55214e6</div><div class="line">tenant_id: 3a55e7b5f5504649a2dfde7147383d02</div><div class="line">extra_dhcp_opts: []</div><div class="line">binding:vnic_type: normal</div><div class="line">device_owner: compute:az_compute</div><div class="line">mac_address: fa:16:3e:52:7a:f2</div><div class="line">fixed_ips: [&#123;u<span class="string">'subnet_id'</span>: u<span class="string">'94bf94c0-6568-4520-aee3-d12b5e472128'</span>, u<span class="string">'ip_address'</span>: u<span class="string">'10.0.0.2'</span>&#125;]</div><div class="line">id: f47c62b0-dbd2-4faa-9cdd-8abc886ce08f</div><div class="line">security_groups: [u<span class="string">'7c2b801b-4590-4a1f-9837-1cceb7f6d1d0'</span>]</div><div class="line">device_id: c3522974-8a08-481c-87b5-fe3822f5c89c</div><div class="line"><span class="comment">## port_id = c4493802-4344-42bd-87a6-1b783f88609a</span></div><div class="line">status: ACTIVE</div><div class="line">name:</div><div class="line">allowed_address_pairs: []</div><div class="line">admin_state_up: True</div><div class="line">network_id: ea3928dc-b1fd-4a1a-940e-82b8c55214e6</div><div class="line">tenant_id: 3a55e7b5f5504649a2dfde7147383d02</div><div class="line">extra_dhcp_opts: []</div><div class="line">binding:vnic_type: normal</div><div class="line">device_owner: compute:az_compute</div><div class="line">mac_address: fa:16:3e:94:84:90</div><div class="line">fixed_ips: [&#123;u<span class="string">'subnet_id'</span>: u<span class="string">'94bf94c0-6568-4520-aee3-d12b5e472128'</span>, u<span class="string">'ip_address'</span>: u<span class="string">'10.0.0.4'</span>&#125;]</div><div class="line">id: c4493802-4344-42bd-87a6-1b783f88609a</div><div class="line">security_groups: [u<span class="string">'7c2b801b-4590-4a1f-9837-1cceb7f6d1d0'</span>]</div><div class="line">device_id: 9365c842-9228-44a6-88ad-33d7389cda5f</div></pre></td></tr></table></figure>
<h5 id="sh"><a href="#sh" class="headerlink" title="sh"></a>sh</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; sh cmd</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;执行系统命令。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div></pre></td><td class="code"><pre><div class="line">EasyOVS&gt; sh ls -l</div><div class="line">total 48</div><div class="line">drwxr-xr-x. 2 root root 4096 Apr  1 14:34 bin</div><div class="line">drwxr-xr-x. 5 root root 4096 Apr  1 14:56 build</div><div class="line">drwxr-xr-x. 2 root root 4096 Apr  1 14:56 dist</div><div class="line">drwxr-xr-x. 2 root root 4096 Apr  1 14:09 doc</div><div class="line">drwxr-xr-x. 4 root root 4096 Apr  1 14:56 easyovs</div><div class="line">-rw-r--r--. 1 root root  660 Apr  1 14:56 easyovs.1</div><div class="line">drwxr-xr-x. 2 root root 4096 Apr  1 14:56 easyovs.egg-info</div><div class="line">-rw-r--r--. 1 root root 2214 Apr  1 14:53 INSTALL</div><div class="line">-rw-r--r--. 1 root root 1194 Apr  1 14:53 Makefile</div><div class="line">-rw-r--r--. 1 root root 3836 Apr  1 14:53 README.md</div><div class="line">-rw-r--r--. 1 root root 1177 Apr  1 14:53 setup.py</div><div class="line">drwxr-xr-x. 2 root root 4096 Apr  1 14:09 util</div></pre></td></tr></table></figure>
<h5 id="quit"><a href="#quit" class="headerlink" title="quit"></a>quit</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;输入 ^d 或者 quit 命令来退出程序。</p>
<h4 id="参数"><a href="#参数" class="headerlink" title="参数"></a>参数</h4><h5 id="h"><a href="#h" class="headerlink" title="-h"></a>-h</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;显示帮助信息。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div></pre></td><td class="code"><pre><div class="line">$ easyovs -h</div><div class="line">Usage: easyovs [options]</div><div class="line">(<span class="built_in">type</span> easyovs -h <span class="keyword">for</span> details)</div><div class="line"></div><div class="line">The easyovs utility creates operation CLI from the <span class="built_in">command</span> line. It can run</div><div class="line">given commands, invoke the EasyOVS CLI, and run tests.</div><div class="line"></div><div class="line">Options:</div><div class="line">  -h, --<span class="built_in">help</span>            show this <span class="built_in">help</span> message and <span class="built_in">exit</span></div><div class="line">  -c, --clean           clean and <span class="built_in">exit</span></div><div class="line">  -m CMD, --cmd=CMD     Run customized commands <span class="keyword">for</span> tests.</div><div class="line">  -v VERBOSITY, --verbosity=VERBOSITY</div><div class="line">                        info|warning|critical|error|debug|output</div><div class="line">  --version</div></pre></td></tr></table></figure>
<h5 id="c"><a href="#c" class="headerlink" title="-c"></a>-c</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;进行环境清理。</p>
<h5 id="m"><a href="#m" class="headerlink" title="-m"></a>-m</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;不进入 CLI，直接执行给定的命令，显示结果。 例如</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">$ sudo easyovs -m <span class="string">"show br-int"</span></div><div class="line">Intf                Port        Vlan    Type        vmIP            vmMAC</div><div class="line">qvof47c62b0-db      2           1                   10.0.0.2        fa:16:3e:52:7a:f2</div><div class="line">qvoc4493802-43      3           1                   10.0.0.4        fa:16:3e:94:84:90</div><div class="line">br-int              LOCAL               internal</div><div class="line">patch-tun           6                   patch</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;例如</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div></pre></td><td class="code"><pre><div class="line">$ sudo easyovs -m <span class="string">'br-int dump'</span></div><div class="line">ID TAB PKT       PRI   MATCH                                                       ACT</div><div class="line">0  0   30        1     *                                                           NORMAL</div><div class="line">1  23  0         0     *                                                           drop</div></pre></td></tr></table></figure>
<h5 id="v"><a href="#v" class="headerlink" title="-v"></a>-v</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;设置输出信息的日志级别，包括 debug，info，warn，error 等，方便进行调试。</p>
<h5 id="–version"><a href="#–version" class="headerlink" title="–version"></a>–version</h5><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;显示版本信息。</p>
<h2 id="参考"><a href="#参考" class="headerlink" title="参考"></a>参考</h2><ul>
<li><a href="http://openstack.redhat.com/Networking_in_too_much_detail" target="_blank" rel="external">http://openstack.redhat.com/Networking_in_too_much_detail</a></li>
<li><a href="http://web.archive.org/web/20150215214007/http://masimum.inf.um.es/fjrm/2013/12/26/the-journey-of-a-packet-within-an-openstack-cloud" target="_blank" rel="external">http://web.archive.org/web/20150215214007/http://masimum.inf.um.es/fjrm/2013/12/26/the-journey-of-a-packet-within-an-openstack-cloud</a></li>
<li><a href="http://packetpushers.net/openstack-quantum-network-implementation-in-linux/" target="_blank" rel="external">http://packetpushers.net/openstack-quantum-network-implementation-in-linux/</a></li>
<li><a href="http://blog.scottlowe.org/2013/09/04/introducing-linux-network-namespaces/" target="_blank" rel="external">http://blog.scottlowe.org/2013/09/04/introducing-linux-network-namespaces/</a></li>
<li><a href="http://assafmuller.wordpress.com/2013/10/14/gre-tunnels-in-openstack-neutron/" target="_blank" rel="external">http://assafmuller.wordpress.com/2013/10/14/gre-tunnels-in-openstack-neutron/</a></li>
<li><a href="http://lwn.net/Articles/580893/" target="_blank" rel="external">http://lwn.net/Articles/580893/</a></li>
</ul>
<h2 id="附：安装配置"><a href="#附：安装配置" class="headerlink" title="附：安装配置"></a>附：安装配置</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;参考 <a href="https://github.com/yeasy/openstack-tool。" target="_blank" rel="external">https://github.com/yeasy/openstack-tool。</a></p>
<h2 id="DevStack-安装-OpenStack-多节点（Juno-Neutron-ML2-VXLAN）"><a href="#DevStack-安装-OpenStack-多节点（Juno-Neutron-ML2-VXLAN）" class="headerlink" title="DevStack 安装 OpenStack 多节点（Juno+Neutron+ML2+VXLAN）"></a>DevStack 安装 OpenStack 多节点（Juno+Neutron+ML2+VXLAN）</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;目前安装 OpenStack 常见的方案有 Redhat 的 RDO 和社区的 DevStack。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;当然，也可以手动安装，可以参考：github.com/ChaimaGhribi/OpenStack-Juno-Installation/blob/master/OpenStack-Juno-Installation.rst</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;其中，RDO 功能比较强大，运行也稳定，可以在一个节点上通过一个 answer 文件直接部署多个节点，搭建一套 OpenStack 环境。但是可惜，在 Ubuntu 上还不支持。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;DevStack 支持 Ubuntu、Fedora 等环境，需要在每个节点上单独执行，适合进行实验。目前常见的教程一般都是讲解 DevStack 单节点安装。本文讲解最新的 Juno 版本在多节点上的安装过程。</p>
<h2 id="网络环境"><a href="#网络环境" class="headerlink" title="网络环境"></a>网络环境</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;两台机器，分为控制节点（同时也作为网络节点）和计算节点。</p>
<h3 id="控制节点"><a href="#控制节点" class="headerlink" title="控制节点"></a>控制节点</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;eth0: 9.186.100.77/24 作为管理网络（同时也是公共网络）。 eth1: 10.0.100.77/24 作为内部网络接口。</p>
<h3 id="计算节点-4"><a href="#计算节点-4" class="headerlink" title="计算节点"></a>计算节点</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;eth0: 9.186.100.88/24 作为管理网络（同时也是公共网络）。 eth1: 10.0.100.88/24 作为内部网络接口。</p>
<h2 id="配置-stack-用户"><a href="#配置-stack-用户" class="headerlink" title="配置 stack 用户"></a>配置 stack 用户</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;创建 stack 用户</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">sudo groupadd stack</div><div class="line">sudo useradd -g stack -s /bin/bash -d /opt/stack -m stack</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;添加 stack 用户权限。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo <span class="built_in">echo</span> <span class="string">"stack ALL=(ALL) NOPASSWD: ALL"</span> &gt;&gt; /etc/sudoers</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;切换到 stack 用户</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo su - stack</div></pre></td></tr></table></figure>
<h2 id="下载代码"><a href="#下载代码" class="headerlink" title="下载代码"></a>下载代码</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;下载 devstack 代码，并切换到 stable/juno 分支。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">sudo apt-get install git -y</div><div class="line">git <span class="built_in">clone</span> https://git.openstack.org/openstack-dev/devstack -b stable/juno</div></pre></td></tr></table></figure>
<h2 id="编写运行配置文件"><a href="#编写运行配置文件" class="headerlink" title="编写运行配置文件"></a>编写运行配置文件</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;在 devstack 根目录下，编写 local.conf。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;控制节点的 local.conf</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div></pre></td><td class="code"><pre><div class="line">[[<span class="built_in">local</span>|localrc]]</div><div class="line"></div><div class="line">MULTI_HOST=<span class="literal">true</span></div><div class="line">HOST_IP=9.186.100.77 <span class="comment"># management network</span></div><div class="line"></div><div class="line">FIXED_RANGE=10.0.0.0/24 <span class="comment">#tenant private network</span></div><div class="line">NETWORK_GATEWAY=10.0.0.1</div><div class="line"><span class="comment">#FIXED_NETWORK_SIZE=4096</span></div><div class="line"></div><div class="line">PUBLIC_INTERFACE=eth0  <span class="comment">#public network</span></div><div class="line">FLOATING_RANGE=9.186.100.128/25</div><div class="line">PUBLIC_NETWORK_GATEWAY=9.186.100.1</div><div class="line"></div><div class="line">TUNNEL_ENDPOINT_IP=10.0.100.77 <span class="comment"># data network endpoint for vxlan</span></div><div class="line"></div><div class="line">LOGFILE=/opt/stack/logs/stack.sh.log</div><div class="line"></div><div class="line"><span class="comment"># Credentials</span></div><div class="line">ADMIN_PASSWORD=admin</div><div class="line">MYSQL_PASSWORD=secret</div><div class="line">RABBIT_PASSWORD=secret</div><div class="line">SERVICE_PASSWORD=secret</div><div class="line">SERVICE_TOKEN=abcdefghijklmnopqrstuvwxyz</div><div class="line"></div><div class="line"><span class="comment"># enable neutron-ml2-vxlan</span></div><div class="line">disable_service n-net</div><div class="line">enable_service q-svc,q-agt,q-dhcp,q-l3,q-meta,q-metering,q-lbaas,q-fwaas,q-vpn,neutron,tempest,heat</div><div class="line"></div><div class="line"><span class="comment">#OFFLINE=True</span></div><div class="line"></div><div class="line">LOG_COLOR=False</div><div class="line">LOGDIR=<span class="variable">$DEST</span>/logs</div><div class="line">SCREEN_LOGDIR=<span class="variable">$LOGDIR</span>/screen</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;计算节点的 local.conf</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div><div class="line">7</div><div class="line">8</div><div class="line">9</div><div class="line">10</div><div class="line">11</div><div class="line">12</div><div class="line">13</div><div class="line">14</div><div class="line">15</div><div class="line">16</div><div class="line">17</div><div class="line">18</div><div class="line">19</div><div class="line">20</div><div class="line">21</div><div class="line">22</div><div class="line">23</div><div class="line">24</div><div class="line">25</div><div class="line">26</div><div class="line">27</div><div class="line">28</div><div class="line">29</div><div class="line">30</div><div class="line">31</div><div class="line">32</div><div class="line">33</div><div class="line">34</div><div class="line">35</div><div class="line">36</div><div class="line">37</div><div class="line">38</div><div class="line">39</div><div class="line">40</div><div class="line">41</div><div class="line">42</div><div class="line">43</div><div class="line">44</div></pre></td><td class="code"><pre><div class="line">[[<span class="built_in">local</span>|localrc]]</div><div class="line">MULTI_HOST=<span class="literal">true</span></div><div class="line"></div><div class="line">HOST_IP=9.186.100.88 <span class="comment"># management network</span></div><div class="line"></div><div class="line">FIXED_RANGE=10.0.0.0/24 <span class="comment"># tenant private network</span></div><div class="line">NETWORK_GATEWAY=10.0.0.1</div><div class="line"><span class="comment">#FIXED_NETWORK_SIZE=4096</span></div><div class="line"></div><div class="line">TUNNEL_ENDPOINT_IP=10.0.100.88 <span class="comment"># data network endpoint for vxlan</span></div><div class="line"></div><div class="line"><span class="comment"># Credentials</span></div><div class="line">ADMIN_PASSWORD=admin</div><div class="line">MYSQL_PASSWORD=secret</div><div class="line">RABBIT_PASSWORD=secret</div><div class="line">SERVICE_PASSWORD=secret</div><div class="line">SERVICE_TOKEN=abcdefghijklmnopqrstuvwxyz</div><div class="line"></div><div class="line"><span class="comment"># Service information</span></div><div class="line">SERVICE_HOST=9.186.100.77</div><div class="line">MYSQL_HOST=<span class="variable">$SERVICE_HOST</span></div><div class="line">RABBIT_HOST=<span class="variable">$SERVICE_HOST</span></div><div class="line">GLANCE_HOSTPORT=<span class="variable">$SERVICE_HOST</span>:9292</div><div class="line">Q_HOST=<span class="variable">$SERVICE_HOST</span></div><div class="line">KEYSTONE_AUTH_HOST=<span class="variable">$SERVICE_HOST</span></div><div class="line">KEYSTONE_SERVICE_HOST=<span class="variable">$SERVICE_HOST</span></div><div class="line"></div><div class="line">CEILOMETER_BACKEND=mongodb</div><div class="line">DATABASE_TYPE=mysql</div><div class="line"></div><div class="line">ENABLED_SERVICES=n-cpu,q-agt,neutron</div><div class="line"></div><div class="line"><span class="comment"># vnc config</span></div><div class="line">NOVA_VNC_ENABLED=True</div><div class="line">NOVNCPROXY_URL=<span class="string">"http://<span class="variable">$SERVICE_HOST</span>:6080/vnc_auto.html"</span></div><div class="line">VNCSERVER_LISTEN=<span class="variable">$HOST_IP</span></div><div class="line">VNCSERVER_PROXYCLIENT_ADDRESS=<span class="variable">$VNCSERVER_LISTEN</span></div><div class="line"></div><div class="line"><span class="comment">#OFFLINE=True</span></div><div class="line"></div><div class="line">LOG_COLOR=False</div><div class="line">LOGDIR=<span class="variable">$DEST</span>/logs</div><div class="line">SCREEN_LOGDIR=<span class="variable">$LOGDIR</span>/screen</div><div class="line"><span class="comment">#LOGFILE=/opt/stack/logs/stack.sh.log</span></div></pre></td></tr></table></figure>
<h2 id="执行配置"><a href="#执行配置" class="headerlink" title="执行配置"></a>执行配置</h2><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;执行命令。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">./stack.sh</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;会输出各项操作的结果。日志会写到 stack.sh.log 文件。成功后最后会打印出相关信息。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;控制节点上</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div><div class="line">3</div><div class="line">4</div><div class="line">5</div><div class="line">6</div></pre></td><td class="code"><pre><div class="line">Horizon is now available at http://9.186.100.77/</div><div class="line">Keystone is serving at http://9.186.100.77:5000/v2.0/</div><div class="line">Examples on using novaclient <span class="built_in">command</span> line is <span class="keyword">in</span> exercise.sh</div><div class="line">The default users are: admin and demo</div><div class="line">The password: admin</div><div class="line">This is your host ip: 9.186.100.77</div></pre></td></tr></table></figure>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;执行成功后，在控制节点上，将物理网卡 eth0 绑定到 br-ex 网桥上。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo ifconfig eth0 0.0.0.0; sudo ovs-vsctl add-port br-ex eth0; sudo ifconfig br-ex 9.186.100.77/24; sudo route add default gw 9.186.100.1</div></pre></td></tr></table></figure>
<h2 id="其它事项"><a href="#其它事项" class="headerlink" title="其它事项"></a>其它事项</h2><h3 id="停止服务"><a href="#停止服务" class="headerlink" title="停止服务"></a>停止服务</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;停止 OpenStack 的各个服务（DevStack 默认每个服务都在一个 screen 中以进程方式运行，可以通过 screen -x stack 进入查看）。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">./unstack.sh</div></pre></td></tr></table></figure>
<h3 id="清除安装。"><a href="#清除安装。" class="headerlink" title="清除安装。"></a>清除安装。</h3><figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">./clean.sh</div></pre></td></tr></table></figure>
<h3 id="手动清除"><a href="#手动清除" class="headerlink" title="手动清除"></a>手动清除</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;有时候有些文件可能清除不干净，手动执行</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div><div class="line">2</div></pre></td><td class="code"><pre><div class="line">sudo rm -rf /etc/libvirt/qemu/inst*</div><div class="line">sudo virsh list | grep inst | awk <span class="string">'&#123;print $1&#125;'</span> | xargs -n1 virsh destroy</div></pre></td></tr></table></figure>
<h3 id="依赖版本"><a href="#依赖版本" class="headerlink" title="依赖版本"></a>依赖版本</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;安装过程中可能会报某些包版本不满足的问题，手动安装后。</p>
<p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;例如报 six 包版本过低。</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><div class="line">1</div></pre></td><td class="code"><pre><div class="line">sudo pip install six --upgrade</div></pre></td></tr></table></figure>
<h3 id="screen-操作"><a href="#screen-操作" class="headerlink" title="screen 操作"></a>screen 操作</h3><p>&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;ctrl+a+” 显示 session 列表。 ctrl+a+d 挂起到后台。</p>

      
    </div>

    <div>
      
        

      
    </div>

    <div>
      
        

      
    </div>

    <div>
      
        

      
    </div>

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/OpenStack/" rel="tag"># OpenStack</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2017/09/03/OpenStack/3. keystone整体架构与功能/" rel="next" title="keystone整体架构与功能">
                <i class="fa fa-chevron-left"></i> keystone整体架构与功能
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2017/09/03/Php/1. LAMP安装php/" rel="prev" title="LAMP安装php">
                LAMP安装php <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          
  <div class="comments" id="comments">
    
  </div>


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap" >
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/images/0.png"
               alt="失落的乐章" />
          <p class="site-author-name" itemprop="name">失落的乐章</p>
           
              <p class="site-description motion-element" itemprop="description">失落的乐章的Blog</p>
          
        </div>
        <nav class="site-state motion-element">

          
            <div class="site-state-item site-state-posts">
              <a href="/">
                <span class="site-state-item-count">627</span>
                <span class="site-state-item-name">日志</span>
              </a>
            </div>
          

          

          
            
            
            <div class="site-state-item site-state-tags">
              <a href="/tags/index.html">
                <span class="site-state-item-count">38</span>
                <span class="site-state-item-name">标签</span>
              </a>
            </div>
          

        </nav>

        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/hcldirgit" target="_blank" title="GitHub">
                  
                    <i class="fa fa-fw fa-github"></i>
                  
                  GitHub
                </a>
              </span>
            
          
        </div>

        
        

        
        

        


      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#概述"><span class="nav-number">1.</span> <span class="nav-text">概述</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#涉及的-Linux-网络技术"><span class="nav-number">1.1.</span> <span class="nav-text">涉及的 Linux 网络技术</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#基本概念"><span class="nav-number">2.</span> <span class="nav-text">基本概念</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#网络"><span class="nav-number">2.1.</span> <span class="nav-text">网络</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#子网"><span class="nav-number">2.2.</span> <span class="nav-text">子网</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#端口"><span class="nav-number">2.3.</span> <span class="nav-text">端口</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#抽象系统架构"><span class="nav-number">2.4.</span> <span class="nav-text">抽象系统架构</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#GRE-模式"><span class="nav-number">3.</span> <span class="nav-text">GRE 模式</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#计算节点"><span class="nav-number">3.1.</span> <span class="nav-text">计算节点</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#qbr"><span class="nav-number">3.2.</span> <span class="nav-text">qbr</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#br-int"><span class="nav-number">3.3.</span> <span class="nav-text">br-int</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#br-tun"><span class="nav-number">3.4.</span> <span class="nav-text">br-tun</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-0"><span class="nav-number">3.5.</span> <span class="nav-text">表 0</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-1"><span class="nav-number">3.6.</span> <span class="nav-text">表 1</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-2"><span class="nav-number">3.7.</span> <span class="nav-text">表 2</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-3"><span class="nav-number">3.8.</span> <span class="nav-text">表 3</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-10"><span class="nav-number">3.9.</span> <span class="nav-text">表 10</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-20"><span class="nav-number">3.10.</span> <span class="nav-text">表 20</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表21"><span class="nav-number">3.11.</span> <span class="nav-text">表21</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#网络节点"><span class="nav-number">4.</span> <span class="nav-text">网络节点</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#br-tun-1"><span class="nav-number">4.1.</span> <span class="nav-text">br-tun</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#br-int-1"><span class="nav-number">4.2.</span> <span class="nav-text">br-int</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#网络名字空间"><span class="nav-number">4.3.</span> <span class="nav-text">网络名字空间</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#dhcp-服务"><span class="nav-number">4.4.</span> <span class="nav-text">dhcp 服务</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#router服务"><span class="nav-number">4.5.</span> <span class="nav-text">router服务</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#br-ex"><span class="nav-number">4.6.</span> <span class="nav-text">br-ex</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#VLAN-模式"><span class="nav-number">5.</span> <span class="nav-text">VLAN 模式</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#计算节点-1"><span class="nav-number">5.1.</span> <span class="nav-text">计算节点</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#br-int-2"><span class="nav-number">5.1.1.</span> <span class="nav-text">br-int</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#br-eth1"><span class="nav-number">5.1.2.</span> <span class="nav-text">br-eth1</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#网络节点-1"><span class="nav-number">5.2.</span> <span class="nav-text">网络节点</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#br-eth1-1"><span class="nav-number">5.2.1.</span> <span class="nav-text">br-eth1</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#br-int-3"><span class="nav-number">5.2.2.</span> <span class="nav-text">br-int</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#br-ex-1"><span class="nav-number">5.2.3.</span> <span class="nav-text">br-ex</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#VXLAN-模式"><span class="nav-number">6.</span> <span class="nav-text">VXLAN 模式</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#计算节点-2"><span class="nav-number">6.1.</span> <span class="nav-text">计算节点</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#br-int-4"><span class="nav-number">6.1.1.</span> <span class="nav-text">br-int</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#br-tun-2"><span class="nav-number">6.1.2.</span> <span class="nav-text">br-tun</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#表-0-1"><span class="nav-number">6.1.3.</span> <span class="nav-text">表 0</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#表-2-1"><span class="nav-number">6.1.4.</span> <span class="nav-text">表 2</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#表-3-1"><span class="nav-number">6.1.5.</span> <span class="nav-text">表 3</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#表-4"><span class="nav-number">6.1.6.</span> <span class="nav-text">表 4</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#表-10-1"><span class="nav-number">6.1.7.</span> <span class="nav-text">表 10</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#表-20-1"><span class="nav-number">6.1.8.</span> <span class="nav-text">表 20</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#表-22"><span class="nav-number">6.1.9.</span> <span class="nav-text">表 22</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#网络节点-2"><span class="nav-number">7.</span> <span class="nav-text">网络节点</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#br-tun-3"><span class="nav-number">7.1.</span> <span class="nav-text">br-tun</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-0-2"><span class="nav-number">7.2.</span> <span class="nav-text">表 0</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-2-2"><span class="nav-number">7.3.</span> <span class="nav-text">表 2</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-3-2"><span class="nav-number">7.4.</span> <span class="nav-text">表 3</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-4-1"><span class="nav-number">7.5.</span> <span class="nav-text">表 4</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-10-2"><span class="nav-number">7.6.</span> <span class="nav-text">表 10</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-20-2"><span class="nav-number">7.7.</span> <span class="nav-text">表 20</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#表-22-1"><span class="nav-number">7.8.</span> <span class="nav-text">表 22</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#br-int-5"><span class="nav-number">7.9.</span> <span class="nav-text">br-int</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#br-ex-2"><span class="nav-number">7.10.</span> <span class="nav-text">br-ex</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#网络名字空间-1"><span class="nav-number">8.</span> <span class="nav-text">网络名字空间</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#DHCP-服务"><span class="nav-number">8.1.</span> <span class="nav-text">DHCP 服务</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#路由服务"><span class="nav-number">8.2.</span> <span class="nav-text">路由服务</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安全组"><span class="nav-number">8.3.</span> <span class="nav-text">安全组</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#INPUT"><span class="nav-number">8.4.</span> <span class="nav-text">INPUT</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#OUTPUT"><span class="nav-number">8.5.</span> <span class="nav-text">OUTPUT</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#FORWARD"><span class="nav-number">8.6.</span> <span class="nav-text">FORWARD</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#整体逻辑"><span class="nav-number">8.7.</span> <span class="nav-text">整体逻辑</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#快速查找安全组规则"><span class="nav-number">9.</span> <span class="nav-text">快速查找安全组规则</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#其它"><span class="nav-number">9.1.</span> <span class="nav-text">其它</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#负载均衡即服务"><span class="nav-number">10.</span> <span class="nav-text">负载均衡即服务</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#典型场景"><span class="nav-number">10.1.</span> <span class="nav-text">典型场景</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#实现细节"><span class="nav-number">10.2.</span> <span class="nav-text">实现细节</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#qlbaas-命名空间"><span class="nav-number">10.3.</span> <span class="nav-text">qlbaas 命名空间</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#HAProxy"><span class="nav-number">10.4.</span> <span class="nav-text">HAProxy</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#其它问题"><span class="nav-number">10.5.</span> <span class="nav-text">其它问题</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#防火墙即服务"><span class="nav-number">11.</span> <span class="nav-text">防火墙即服务</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#典型场景-1"><span class="nav-number">11.1.</span> <span class="nav-text">典型场景</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#实现细节-1"><span class="nav-number">11.2.</span> <span class="nav-text">实现细节</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#其它问题-1"><span class="nav-number">11.3.</span> <span class="nav-text">其它问题</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#分布式路由"><span class="nav-number">12.</span> <span class="nav-text">分布式路由</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#典型场景-2"><span class="nav-number">12.1.</span> <span class="nav-text">典型场景</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#东西向"><span class="nav-number">12.1.1.</span> <span class="nav-text">东西向</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#同一机器"><span class="nav-number">12.1.1.1.</span> <span class="nav-text">同一机器</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#不同机器"><span class="nav-number">12.1.1.2.</span> <span class="nav-text">不同机器</span></a></li></ol></li><li class="nav-item nav-level-4"><a class="nav-link" href="#南北向"><span class="nav-number">12.1.2.</span> <span class="nav-text">南北向</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#无-floating-IP"><span class="nav-number">12.1.2.1.</span> <span class="nav-text">无 floating IP</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#有-floating-IP"><span class="nav-number">12.1.2.2.</span> <span class="nav-text">有 floating IP</span></a></li></ol></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#网络节点-3"><span class="nav-number">12.2.</span> <span class="nav-text">网络节点</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#计算节点-3"><span class="nav-number">12.3.</span> <span class="nav-text">计算节点</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#东西流量"><span class="nav-number">12.3.1.</span> <span class="nav-text">东西流量</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#南北流量"><span class="nav-number">12.3.2.</span> <span class="nav-text">南北流量</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#流表规则"><span class="nav-number">12.4.</span> <span class="nav-text">流表规则</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#所有对网关的-arp-request-不能出到外部网络"><span class="nav-number">12.4.1.</span> <span class="nav-text">所有对网关的 arp request 不能出到外部网络</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#所有发往网关的包也不能出到外部网络"><span class="nav-number">12.4.2.</span> <span class="nav-text">所有发往网关的包也不能出到外部网络</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#所有从路由器接口路由出去的包，需要修改源-mac-为特殊-mac"><span class="nav-number">12.4.3.</span> <span class="nav-text">所有从路由器接口路由出去的包，需要修改源 mac 为特殊 mac</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#同样，收到外部特殊源-mac-的包要修改为正确的路由器接口-mac"><span class="nav-number">12.4.4.</span> <span class="nav-text">同样，收到外部特殊源 mac 的包要修改为正确的路由器接口 mac</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#注意事项"><span class="nav-number">12.5.</span> <span class="nav-text">注意事项</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#配置"><span class="nav-number">12.6.</span> <span class="nav-text">配置</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Neutron-Server"><span class="nav-number">12.6.1.</span> <span class="nav-text">Neutron Server</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#L3-Agent"><span class="nav-number">12.6.2.</span> <span class="nav-text">L3 Agent</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#L2-Agent"><span class="nav-number">12.6.3.</span> <span class="nav-text">L2 Agent</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#实现细节-2"><span class="nav-number">12.7.</span> <span class="nav-text">实现细节</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#添加路由器接口"><span class="nav-number">12.7.1.</span> <span class="nav-text">添加路由器接口</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#删除路由器接口"><span class="nav-number">12.7.2.</span> <span class="nav-text">删除路由器接口</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#启动新的虚拟机节点"><span class="nav-number">12.7.3.</span> <span class="nav-text">启动新的虚拟机节点</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#删除虚拟机节点"><span class="nav-number">12.7.4.</span> <span class="nav-text">删除虚拟机节点</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#实现细节-3"><span class="nav-number">12.8.</span> <span class="nav-text">实现细节</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#integration-网桥规则添加"><span class="nav-number">12.8.1.</span> <span class="nav-text">integration 网桥规则添加</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#tunnel-网桥规则添加"><span class="nav-number">12.8.2.</span> <span class="nav-text">tunnel 网桥规则添加</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#physical-网桥规则添加"><span class="nav-number">12.8.3.</span> <span class="nav-text">physical 网桥规则添加</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#所有网桥上添加-mac-规则"><span class="nav-number">12.8.4.</span> <span class="nav-text">所有网桥上添加 mac 规则</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#外部网络为-vlan-的情况下"><span class="nav-number">12.8.4.1.</span> <span class="nav-text">外部网络为 vlan 的情况下</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#外部网络为-tunnel-的情况下"><span class="nav-number">12.8.4.2.</span> <span class="nav-text">外部网络为 tunnel 的情况下</span></a></li></ol></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#工具"><span class="nav-number">13.</span> <span class="nav-text">工具</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#easyOVS"><span class="nav-number">13.1.</span> <span class="nav-text">easyOVS</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#主要功能一览"><span class="nav-number">13.1.1.</span> <span class="nav-text">主要功能一览</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#安装"><span class="nav-number">13.1.2.</span> <span class="nav-text">安装</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#打开-OpenStack-支持"><span class="nav-number">13.1.3.</span> <span class="nav-text">打开 OpenStack 支持</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#环境变量"><span class="nav-number">13.1.3.1.</span> <span class="nav-text">环境变量</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#配置文件"><span class="nav-number">13.1.3.2.</span> <span class="nav-text">配置文件</span></a></li></ol></li><li class="nav-item nav-level-4"><a class="nav-link" href="#操作命令"><span class="nav-number">13.1.4.</span> <span class="nav-text">操作命令</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#help"><span class="nav-number">13.1.4.1.</span> <span class="nav-text">help</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#list"><span class="nav-number">13.1.4.2.</span> <span class="nav-text">list</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#show"><span class="nav-number">13.1.4.3.</span> <span class="nav-text">show</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#dump"><span class="nav-number">13.1.4.4.</span> <span class="nav-text">dump</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#addflow"><span class="nav-number">13.1.4.5.</span> <span class="nav-text">addflow</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#delflow"><span class="nav-number">13.1.4.6.</span> <span class="nav-text">delflow</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#set"><span class="nav-number">13.1.4.7.</span> <span class="nav-text">set</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#exit"><span class="nav-number">13.1.4.8.</span> <span class="nav-text">exit</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#get"><span class="nav-number">13.1.4.9.</span> <span class="nav-text">get</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#ipt"><span class="nav-number">13.1.4.10.</span> <span class="nav-text">ipt</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#query"><span class="nav-number">13.1.4.11.</span> <span class="nav-text">query</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#sh"><span class="nav-number">13.1.4.12.</span> <span class="nav-text">sh</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#quit"><span class="nav-number">13.1.4.13.</span> <span class="nav-text">quit</span></a></li></ol></li><li class="nav-item nav-level-4"><a class="nav-link" href="#参数"><span class="nav-number">13.1.5.</span> <span class="nav-text">参数</span></a><ol class="nav-child"><li class="nav-item nav-level-5"><a class="nav-link" href="#h"><span class="nav-number">13.1.5.1.</span> <span class="nav-text">-h</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#c"><span class="nav-number">13.1.5.2.</span> <span class="nav-text">-c</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#m"><span class="nav-number">13.1.5.3.</span> <span class="nav-text">-m</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#v"><span class="nav-number">13.1.5.4.</span> <span class="nav-text">-v</span></a></li><li class="nav-item nav-level-5"><a class="nav-link" href="#–version"><span class="nav-number">13.1.5.5.</span> <span class="nav-text">–version</span></a></li></ol></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#参考"><span class="nav-number">14.</span> <span class="nav-text">参考</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#附：安装配置"><span class="nav-number">15.</span> <span class="nav-text">附：安装配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#DevStack-安装-OpenStack-多节点（Juno-Neutron-ML2-VXLAN）"><span class="nav-number">16.</span> <span class="nav-text">DevStack 安装 OpenStack 多节点（Juno+Neutron+ML2+VXLAN）</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#网络环境"><span class="nav-number">17.</span> <span class="nav-text">网络环境</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#控制节点"><span class="nav-number">17.1.</span> <span class="nav-text">控制节点</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#计算节点-4"><span class="nav-number">17.2.</span> <span class="nav-text">计算节点</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#配置-stack-用户"><span class="nav-number">18.</span> <span class="nav-text">配置 stack 用户</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#下载代码"><span class="nav-number">19.</span> <span class="nav-text">下载代码</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#编写运行配置文件"><span class="nav-number">20.</span> <span class="nav-text">编写运行配置文件</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#执行配置"><span class="nav-number">21.</span> <span class="nav-text">执行配置</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#其它事项"><span class="nav-number">22.</span> <span class="nav-text">其它事项</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#停止服务"><span class="nav-number">22.1.</span> <span class="nav-text">停止服务</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#清除安装。"><span class="nav-number">22.2.</span> <span class="nav-text">清除安装。</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#手动清除"><span class="nav-number">22.3.</span> <span class="nav-text">手动清除</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#依赖版本"><span class="nav-number">22.4.</span> <span class="nav-text">依赖版本</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#screen-操作"><span class="nav-number">22.5.</span> <span class="nav-text">screen 操作</span></a></li></ol></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  
  &copy; 
  <span itemprop="copyrightYear">2017</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">失落的乐章</span>
</div>


<div class="powered-by">
  由 <a class="theme-link" href="https://hexo.io">Hexo</a> 强力驱动
</div>

<div class="theme-info">
  主题 -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Muse
  </a>
</div>


        

        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.1"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.1"></script>



  
  

  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.1"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.1"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.1"></script>



  


  




	





  





  





  






  





  

  

  

  

  

  

</body>
</html>
